diff --git a/nix/model-catalog.json b/nix/model-catalog.json index 3c31c2cca..3b4641ee5 100644 --- a/nix/model-catalog.json +++ b/nix/model-catalog.json @@ -1,3 +1,3 @@ { - "revision": "sha256-7fe1b41feaf43a7aad4bc832e6b306c7ced56e6d41bbc724b74e91eeacff6edf" + "revision": "sha256-f4dbabe79cda4b38aee4a761b23fd5eaf526ac2e28fed3c9ee86e37ca0e5d71e" } diff --git a/package-lock.json b/package-lock.json index 74451ff75..5655e8690 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,18 +13,15 @@ "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", - "packages/coding-agent/examples/extensions/sandbox", "packages/coding-agent/examples/extensions/gondolin" ], "devDependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26", "@biomejs/biome": "2.3.5", "@biomejs/wasm-nodejs": "2.3.5", "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", "pnpm": "11.15.1", - "shx": "0.3.4", "typescript": "7.0.2", "yaml": "2.9.0" }, @@ -32,26 +29,6 @@ "node": ">=22.19.0" } }, - "node_modules/@anthropic-ai/sandbox-runtime": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.26.tgz", - "integrity": "sha512-DYV5LSsVMnzq0lbfaYMSpxZPUMAx4+hy343dRss+pVCLIfF62qOhxpYfZ5TmOk1GTDQm5f9wPprMNSStmnsV4w==", - "license": "Apache-2.0", - "dependencies": { - "@pondwader/socks5-server": "^1.0.10", - "@types/lodash-es": "^4.17.12", - "commander": "^12.1.0", - "lodash-es": "^4.17.21", - "shell-quote": "^1.8.3", - "zod": "^3.24.1" - }, - "bin": { - "srt": "dist/cli.js" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/@anthropic-ai/sdk": { "version": "0.129.0", "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.129.0.tgz", @@ -73,6 +50,116 @@ } } }, + "node_modules/@ata-validator/native-darwin-arm64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-arm64/-/native-darwin-arm64-1.61.3.tgz", + "integrity": "sha512-DC+iozKe5mNpsmT4MLuMxG2pIeGPVR3YBYuBmops0AF5UG6eSxg/X0484hW/Nmc2dil54+b4N41hIhS5U9rExg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-darwin-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-darwin-x64/-/native-darwin-x64-1.61.3.tgz", + "integrity": "sha512-q63+k48mWlvrNaUXmtGVxp2NWTl3wXlkkWkc2q4MerHzjIfeFBca30tfTRcyVXtDXPCj8BRJkrI69aJJXk+pLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-gnu/-/native-linux-arm64-gnu-1.61.3.tgz", + "integrity": "sha512-D9OG2oKJd/hW9F8IFoqIfK+SDfyryzSGr5YRVJB39LwvvWNNfh6pxsYpjthEjwqGuJdoBtYIqOUFnjgBqDWh9w==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-arm64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-arm64-musl/-/native-linux-arm64-musl-1.61.3.tgz", + "integrity": "sha512-L1NhRd5/i6v7VQw28gR5L53+Zlo09CnYfXBapgGUC9I3uNppOw56/dihU5cPrd5T/k9VeljosllOP+dWA9cLpA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-gnu": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-gnu/-/native-linux-x64-gnu-1.61.3.tgz", + "integrity": "sha512-Dlp+hcvQqKkAQEyrr7RohCbYoLYHdSjeB5puU3Ju92kSeH4MaF451q0VqwOm3StaMQB3LrLA03GDb4/G3b5gsA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-linux-x64-musl": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-linux-x64-musl/-/native-linux-x64-musl-1.61.3.tgz", + "integrity": "sha512-2TutVxyhzbpUyBFzuGNp4fjyC3Suhi1idFfbk883tPdfcmp2/j0Q5QT5ySk4LPLGrNzV2haqjESZvha/QKctkw==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@ata-validator/native-win32-x64": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/@ata-validator/native-win32-x64/-/native-win32-x64-1.61.3.tgz", + "integrity": "sha512-Ac2fqD70b/lQSuLDdz03xiGPhn2FK7oKXdQ7JbCq9ZYiJBbymhFazMKkRaI+3Y5XKIV3zalAfNGzUJAzMUsXow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@aws-sdk/client-bedrock-runtime": { "version": "3.1127.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1127.0.tgz", @@ -593,6 +680,13 @@ "dev": true, "license": "MIT OR Apache-2.0" }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "dev": true, + "license": "MIT" + }, "node_modules/@cloudflare/kv-asset-handler": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", @@ -620,26 +714,26 @@ } }, "node_modules/@cloudflare/vitest-plugin": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.4.4.tgz", - "integrity": "sha512-nXNIlPiLvYFx/AH4/Bb/yntVkeloYcf1+tBi3l6Gp3TDCDtLxXAugrUtO7eTOWbHP6UIx5f9npy05OrACbZ5wg==", + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@cloudflare/vitest-plugin/-/vitest-plugin-1.4.10.tgz", + "integrity": "sha512-F6klOeT/Wx0gKvZ4zpYVBixP6TjcT4OD/L/RJK3Yq3SwRmd+v96/y037xUEsvGytdUisL9X5Jw3ADuNdvwBHdg==", "dev": true, "license": "MIT", "dependencies": { "cjs-module-lexer": "1.2.3", "esbuild": "0.28.2", - "miniflare": "5.20261012.1-beta", - "wrangler": "4.151.1", - "zod": "4.6.5" + "miniflare": "5.20261023.2-beta", + "wrangler": "4.155.0", + "zod": "4.6.6" }, "peerDependencies": { "vitest": "^4.1.0 || ^5.0.0" } }, "node_modules/@cloudflare/vitest-plugin/node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", "dev": true, "license": "MIT", "funding": { @@ -647,9 +741,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261012.1.tgz", - "integrity": "sha512-dkTL9aY3V4D40b+jBpGPCistWUPFQ/M61wlWi6i1o8MvHFFgEw7EQkcaYEv6Yd3bkPDD/3TFT4hRsMb2xVZSwQ==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20261023.1.tgz", + "integrity": "sha512-lsEx0i7nR0PNG/erUFFHmQmiidEhLfDE5uQmAL9421/y4/ycuxDDOSaxu3VGGYFFPK5sduyUeMmjIlX4CwOBFg==", "cpu": [ "x64" ], @@ -664,9 +758,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261012.1.tgz", - "integrity": "sha512-2/nVGcKXjNNI+DhxKSepTKvFng9H7aN2WXv6Zu67iqgI6JrxypNt5kTPv4ZHosxrcfyKMci/2RfN7GIrN0BitQ==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20261023.1.tgz", + "integrity": "sha512-5e79fHgabdDD8EgmjOhJ1S+xMsf6DbC7fL7IgTBizMnzOp0P2isDn7pYmUxI9AJZE9FVQ4RUilH/I36YNg+GQQ==", "cpu": [ "arm64" ], @@ -681,9 +775,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261012.1.tgz", - "integrity": "sha512-uMe7ONHnLxStD1Ye0YFHBO36uj4cStv5P53rW+5GVWXFT8BRjqVixlH0IqCAHe1Pmcwyd1+7fVoGde89MZSCJg==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20261023.1.tgz", + "integrity": "sha512-VHQ6jKFSihjR2+x1buby8XTdBLLJztVnYiB7OLOTYO8aDXB+c1Rd2410PdgFuFYc4AgFEk6PCJLyCLL5L1KBnw==", "cpu": [ "x64" ], @@ -698,9 +792,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261012.1.tgz", - "integrity": "sha512-I0VMdcXk7ygG5vYDce0ePTBFdnBXC0i9WiBY5mWiv0iTinbfea6/rHqSQ7B1UMpjgrYVs9E0m4bZ9ufsk5GCFA==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20261023.1.tgz", + "integrity": "sha512-ilpRHzfDN8YFRM5CLRKuFw40x/LJBS89yVdhdQmxehlVDuBqjKpJB++zPFbTgKRRFaNupgnXc8RHktXAIhgCmQ==", "cpu": [ "arm64" ], @@ -715,9 +809,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261012.1.tgz", - "integrity": "sha512-c5hQvNx574UEoknaCFvMa/NdLrrYiALNbe6sx1okW8uOiRYWxrFuA8WpXFnBJ5XJypm0TEAHYaUTGv3rRLpY7Q==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20261023.1.tgz", + "integrity": "sha512-279WLVjNsMycAwr0EZNiNEf0l2SY6NUtsN80NwZzbTHzakmiN+WSx4w38muxPm+PAU0oSPPAPXKVD+H6Hya8xw==", "cpu": [ "x64" ], @@ -851,6 +945,10 @@ "resolved": "packages/mcp", "link": true }, + "node_modules/@earendil-works/pi-schema": { + "resolved": "packages/schema", + "link": true + }, "node_modules/@earendil-works/pi-telemetry": { "resolved": "packages/telemetry", "link": true @@ -877,7 +975,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -894,7 +991,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -911,7 +1007,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -928,7 +1023,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -945,7 +1039,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -962,7 +1055,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -979,7 +1071,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -996,7 +1087,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1013,7 +1103,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1030,7 +1119,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1047,7 +1135,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1064,7 +1151,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1081,7 +1167,6 @@ "cpu": [ "mips64el" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1098,7 +1183,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1115,7 +1199,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1132,7 +1215,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1149,7 +1231,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1166,7 +1247,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1183,7 +1263,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1200,7 +1279,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1217,7 +1295,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1234,7 +1311,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1251,7 +1327,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1268,7 +1343,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1285,7 +1359,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1302,7 +1375,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1454,6 +1526,9 @@ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1471,6 +1546,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1488,6 +1566,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1505,6 +1586,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1522,6 +1606,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1539,6 +1626,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1556,6 +1646,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1573,6 +1666,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1590,6 +1686,9 @@ "arm" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1613,6 +1712,9 @@ "arm64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1636,6 +1738,9 @@ "ppc64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1659,6 +1764,9 @@ "riscv64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1682,6 +1790,9 @@ "s390x" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1705,6 +1816,9 @@ "x64" ], "dev": true, + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1728,6 +1842,9 @@ "arm64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1751,6 +1868,9 @@ "x64" ], "dev": true, + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1863,6 +1983,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -1891,6 +2017,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@oh-my-pi/omptype": { + "version": "18.13.3", + "resolved": "https://registry.npmjs.org/@oh-my-pi/omptype/-/omptype-18.13.3.tgz", + "integrity": "sha512-ZLi+kb3ff554n2ndH4iEviVRWHI1t78LZu7qf/NSXkYf4STcl6vEnJ5A2FBhfUaTLVk/HRs09LzyFnySOkwpvQ==", + "dev": true, + "license": "MIT", + "engines": { + "bun": ">=1.4.4", + "node": ">=20" + } + }, "node_modules/@oxc-project/types": { "version": "0.154.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.154.0.tgz", @@ -1901,12 +2038,6 @@ "url": "https://github.com/sponsors/oxc-project" } }, - "node_modules/@pondwader/socks5-server": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", - "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", - "license": "MIT" - }, "node_modules/@poppinss/colors": { "version": "4.1.6", "resolved": "https://registry.npmjs.org/@poppinss/colors/-/colors-4.1.6.tgz", @@ -2441,21 +2572,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/lodash": { - "version": "4.17.24", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.24.tgz", - "integrity": "sha512-gIW7lQLZbue7lRSWEFql49QJJWThrTFFeIMJdp3eH4tKoxm1OvEPg02rm4wCCSHS0cL3/Fizimb35b7k8atwsQ==", - "license": "MIT" - }, - "node_modules/@types/lodash-es": { - "version": "4.17.12", - "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", - "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, "node_modules/@types/ms": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", @@ -2482,6 +2598,26 @@ "@types/retry": "*" } }, + "node_modules/@types/react": { + "version": "19.2.14", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", + "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, "node_modules/@types/retry": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", @@ -2995,6 +3131,41 @@ "node": ">= 14" } }, + "node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/asn1": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", @@ -3014,6 +3185,36 @@ "node": ">=12" } }, + "node_modules/ata-validator": { + "version": "1.61.3", + "resolved": "https://registry.npmjs.org/ata-validator/-/ata-validator-1.61.3.tgz", + "integrity": "sha512-6o3V8lpiVQzzHe6blbQZj9saD0Qxmhdv2TTyO5BgPcUWeP3EZj572f9w1b93E6c+BmUce0QOf7GvwOffRUf3tw==", + "dev": true, + "license": "MIT", + "bin": { + "ata": "bin/ata.js" + }, + "engines": { + "node": ">=20.0.0" + }, + "optionalDependencies": { + "@ata-validator/native-darwin-arm64": "1.61.3", + "@ata-validator/native-darwin-x64": "1.61.3", + "@ata-validator/native-linux-arm64-gnu": "1.61.3", + "@ata-validator/native-linux-arm64-musl": "1.61.3", + "@ata-validator/native-linux-x64-gnu": "1.61.3", + "@ata-validator/native-linux-x64-musl": "1.61.3", + "@ata-validator/native-win32-x64": "1.61.3" + }, + "peerDependencies": { + "yaml": "^2.0.0" + }, + "peerDependenciesMeta": { + "yaml": { + "optional": true + } + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -3201,22 +3402,6 @@ "dev": true, "license": "MIT" }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -3266,6 +3451,13 @@ "node": ">= 8" } }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -3366,16 +3558,6 @@ "url": "https://github.com/sponsors/antfu" } }, - "node_modules/es-errors": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", - "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - } - }, "node_modules/es-module-lexer": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", @@ -3387,7 +3569,7 @@ "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -3471,12 +3653,36 @@ "node": ">=18" } }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, "node_modules/fast-sha256": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", "license": "Unlicense" }, + "node_modules/fast-uri": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.9.tgz", + "integrity": "sha512-hS/0Wt3Ya043oKXs0sIxrMpKQITXhMLIrVAKKXi4gozGC9v7YRWVnXIZzjNPTO02M9BdxLCWqK1yRXgNziVyJQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/fdir": { "version": "6.5.0", "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", @@ -3537,13 +3743,6 @@ "dev": true, "license": "MIT" }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, - "license": "ISC" - }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -3559,16 +3758,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gaxios": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.1.4.tgz", @@ -3616,59 +3805,6 @@ "dev": true, "license": "MIT" }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob/node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "1.1.21", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", - "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/google-auth-library": { "version": "10.6.2", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.6.2.tgz", @@ -3710,19 +3846,6 @@ "node": ">=18" } }, - "node_modules/hasown": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.4.tgz", - "integrity": "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/highlight.js": { "version": "10.7.3", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.7.3.tgz", @@ -3826,18 +3949,6 @@ "node": ">= 4" } }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -3852,32 +3963,6 @@ "dev": true, "license": "ISC" }, - "node_modules/interpret": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", - "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-core-module": { - "version": "2.17.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.17.0.tgz", - "integrity": "sha512-J/vG0zBCbIKOQFfufSwyXdMrsohyJIUNkrnmo6WZGzoM7tr/lsbfW5b2BvisL6zsyMzK9UxV9L6c7AoFbyXHOA==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -3915,6 +4000,13 @@ "node": ">=16" } }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -4219,12 +4311,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "license": "MIT" - }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -4288,16 +4374,16 @@ } }, "node_modules/miniflare": { - "version": "5.20261012.1-beta", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261012.1-beta.tgz", - "integrity": "sha512-8Canhy6g9+vAhqMc0o80YphqYIwpiA3aAp8aX+uDXU7J/qbUeoZzcII9c2V1P4uT/eHD4SB93qet1o1bRY3BuQ==", + "version": "5.20261023.2-beta", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-5.20261023.2-beta.tgz", + "integrity": "sha512-sXpZloQnIUe4raXXdwwqd2ioRHox9a/1B4nB2/p4JWKDwGuVgEK4DugCNgIeQf0iVuSgBItdYpodJe4iwY6SrQ==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", "sharp": "0.35.5", - "undici": "7.29.1", - "workerd": "1.20261012.1", + "undici": "7.30.1", + "workerd": "1.20261023.1", "ws": "8.21.0", "youch": "4.1.0-beta.10" }, @@ -4306,9 +4392,9 @@ } }, "node_modules/miniflare/node_modules/undici": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", - "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", "dev": true, "license": "MIT", "engines": { @@ -4316,12 +4402,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz", + "integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" @@ -4493,16 +4580,6 @@ "integrity": "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA==", "license": "MIT" }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -4512,13 +4589,6 @@ "node": ">=8" } }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, "node_modules/path-to-regexp": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", @@ -4545,10 +4615,6 @@ "resolved": "packages/coding-agent/examples/extensions/gondolin", "link": true }, - "node_modules/pi-extension-sandbox": { - "resolved": "packages/coding-agent/examples/extensions/sandbox", - "link": true - }, "node_modules/pi-extension-with-deps": { "resolved": "packages/coding-agent/examples/extensions/with-deps", "link": true @@ -4742,6 +4808,29 @@ "rc": "cli.js" } }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -4757,38 +4846,14 @@ "node": ">= 6" } }, - "node_modules/rechoir": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", - "integrity": "sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==", - "dev": true, - "dependencies": { - "resolve": "^1.1.6" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/resolve": { - "version": "1.22.12", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.12.tgz", - "integrity": "sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==", + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", "dev": true, "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "node": ">=0.10.0" } }, "node_modules/retry": { @@ -4860,6 +4925,13 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, "node_modules/semver": { "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", @@ -4943,53 +5015,6 @@ "node": ">=8" } }, - "node_modules/shell-quote": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.12.0.tgz", - "integrity": "sha512-PcByqNyT/38F2kDNi006HAMRJaULuBzq/FOsw3qdZvX/GA9W/jamDaRskgHjubHiftXK5sIFxLNkvrXUwcof6Q==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/shelljs": { - "version": "0.8.5", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.5.tgz", - "integrity": "sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "glob": "^7.0.0", - "interpret": "^1.0.0", - "rechoir": "^0.6.2" - }, - "bin": { - "shjs": "bin/shjs" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/shx": { - "version": "0.3.4", - "resolved": "https://registry.npmjs.org/shx/-/shx-0.3.4.tgz", - "integrity": "sha512-N6A9MLVqjxZYcVn8hLmtneQWIJtp8IKzMP4eMnx+nqkvXoqinUPCbUFLp2UcWTEIUONhlk0ewxr/jaVGlc+J+g==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.3", - "shelljs": "^0.8.5" - }, - "bin": { - "shx": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -5134,19 +5159,6 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/tar-fs": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", @@ -5563,9 +5575,9 @@ } }, "node_modules/workerd": { - "version": "1.20261012.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261012.1.tgz", - "integrity": "sha512-tEHoMPsMga0seVjxfsR8Pd3f8IRqYwkq++dTUKI5Gdgj6zuSDluYwisydq+dn90SsIACwBquJUQ1nBpcOqaWTA==", + "version": "1.20261023.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20261023.1.tgz", + "integrity": "sha512-NeofJFyaYNpqKpFRJW3ApuonaerPd6/JWlfTtLDYf2A6ygqrDEFnYgEXmc0hhOxqnJsSM+kXJN6VebPojzRrsg==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", @@ -5576,17 +5588,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20261012.1", - "@cloudflare/workerd-darwin-arm64": "1.20261012.1", - "@cloudflare/workerd-linux-64": "1.20261012.1", - "@cloudflare/workerd-linux-arm64": "1.20261012.1", - "@cloudflare/workerd-windows-64": "1.20261012.1" + "@cloudflare/workerd-darwin-64": "1.20261023.1", + "@cloudflare/workerd-darwin-arm64": "1.20261023.1", + "@cloudflare/workerd-linux-64": "1.20261023.1", + "@cloudflare/workerd-linux-arm64": "1.20261023.1", + "@cloudflare/workerd-windows-64": "1.20261023.1" } }, "node_modules/wrangler": { - "version": "4.151.1", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.151.1.tgz", - "integrity": "sha512-OAsa9UAHZshM6mq2PXcVoyxXgWThAanZMBc/IirtqEWd/BauRtmUKzAniOu/pU0pd3pxKuMZ2vsbF4eo6RiNDA==", + "version": "4.155.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.155.0.tgz", + "integrity": "sha512-gBKtAg2e0abQDvXPRQu611YMTd/cfAaJTOLAs/89tTxFxEeUyC260MRJRK+m3gLmQjDYWFwJv+ErwuJ/GuALpA==", "dev": true, "license": "MIT OR Apache-2.0", "dependencies": { @@ -5594,10 +5606,10 @@ "@cloudflare/unenv-preset": "2.16.2", "blake3-wasm": "2.1.5", "esbuild": "0.28.2", - "miniflare": "5.20261012.1-beta", + "miniflare": "5.20261023.2-beta", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20261012.1" + "workerd": "1.20261023.1" }, "bin": { "cf-wrangler": "bin/cf-wrangler.js", @@ -5611,7 +5623,7 @@ "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^5.20261006.1" + "@cloudflare/workers-types": "^5.20261023.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { @@ -5688,9 +5700,10 @@ } }, "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "version": "4.6.7", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.7.tgz", + "integrity": "sha512-g8W2D2kU6a8untc053fX1uVBKzM7yqAxW/3zwk83+6RyF2+SKOuV+c0wBDhUwzV1PaGapmZH+rmcUzPf+3CGwg==", + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" @@ -5698,11 +5711,11 @@ }, "packages/agent": { "name": "@earendil-works/pi-agent-core", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "devDependencies": { "@types/node": "22.19.19", @@ -5721,19 +5734,19 @@ }, "packages/ai": { "name": "@earendil-works/pi-ai", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "bin": { "pi-ai": "dist/cli.js" @@ -5741,6 +5754,7 @@ "devDependencies": { "@types/node": "22.19.19", "canvas": "3.2.3", + "typebox": "1.3.27", "vitest": "4.1.12" }, "engines": { @@ -5809,31 +5823,49 @@ }, "packages/chord": { "name": "@earendil-works/chord", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-telemetry": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0" }, "devDependencies": { + "@types/react": "19.2.14", + "@types/react-dom": "19.2.3", "@types/ws": "8.18.2", - "shx": "0.3.4", + "react": "19.3.0", + "react-dom": "19.3.0", "vitest": "4.1.12", "ws": "8.21.0" }, "engines": { "node": ">=22.19.0" + }, + "peerDependencies": { + "esbuild": "0.28.2", + "react": "19.3.0", + "ws": "8.21.0" + }, + "peerDependenciesMeta": { + "esbuild": { + "optional": true + }, + "react": { + "optional": true + }, + "ws": { + "optional": true + } } }, "packages/codemode": { "name": "@earendil-works/pi-codemode", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "quickjs-wasi": "3.6.2" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5842,14 +5874,14 @@ }, "packages/coding-agent": { "name": "@earendil-works/pi-coding-agent", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -5860,7 +5892,7 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", @@ -5878,7 +5910,6 @@ "@types/proper-lockfile": "4.1.4", "@types/semver": "7.7.1", "marked": "18.0.11", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5887,32 +5918,25 @@ }, "packages/coding-agent/examples/extensions/custom-provider-anthropic": { "name": "pi-extension-custom-provider-anthropic", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@anthropic-ai/sdk": "0.52.0" } }, "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo": { "name": "pi-extension-custom-provider-gitlab-duo", - "version": "1.4.0" + "version": "1.5.0" }, "packages/coding-agent/examples/extensions/gondolin": { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@earendil-works/gondolin": "0.14.0" } }, - "packages/coding-agent/examples/extensions/sandbox": { - "name": "pi-extension-sandbox", - "version": "1.4.0", - "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26" - } - }, "packages/coding-agent/examples/extensions/with-deps": { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "ms": "2.1.3" }, @@ -5966,21 +5990,20 @@ }, "packages/durable": { "name": "@earendil-works/pi-durable", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@silvia-odwyer/photon": "0.3.3", - "diff": "8.0.4", - "typebox": "1.3.27" + "diff": "8.0.4" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.4.4", + "@cloudflare/vitest-plugin": "1.4.10", "fake-indexeddb": "6.2.6", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -5989,14 +6012,13 @@ }, "packages/env": { "name": "@earendil-works/pi-env", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-durable": "1.4.0" + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-durable": "1.5.0" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { @@ -6005,36 +6027,52 @@ }, "packages/evals": { "name": "@earendil-works/pi-evals", - "version": "1.4.0", + "version": "1.5.0", "devDependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-coding-agent": "1.4.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-coding-agent": "1.5.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" } }, "packages/mcp": { "name": "@earendil-works/pi-mcp", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" }, "devDependencies": { "@types/cross-spawn": "6.0.6", - "shx": "0.3.4", "vitest": "4.1.12" }, "engines": { "node": ">=22.19.0" } }, + "packages/schema": { + "name": "@earendil-works/pi-schema", + "version": "1.5.0", + "license": "MIT", + "devDependencies": { + "@cfworker/json-schema": "4.1.1", + "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", + "typebox": "1.3.27", + "vitest": "4.1.12", + "zod": "4.6.7" + }, + "engines": { + "node": ">=22.19.0" + } + }, "packages/telemetry": { "name": "@earendil-works/pi-telemetry", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "devDependencies": { "@types/node": "22.19.19", @@ -6046,7 +6084,7 @@ }, "packages/tui": { "name": "@earendil-works/pi-tui", - "version": "1.4.0", + "version": "1.5.0", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", diff --git a/package.json b/package.json index 04462a1d5..daa608b30 100644 --- a/package.json +++ b/package.json @@ -8,17 +8,16 @@ "packages/coding-agent/examples/extensions/with-deps", "packages/coding-agent/examples/extensions/custom-provider-anthropic", "packages/coding-agent/examples/extensions/custom-provider-gitlab-duo", - "packages/coding-agent/examples/extensions/sandbox", "packages/coding-agent/examples/extensions/gondolin" ], "scripts": { "clean": "npm run clean --workspaces", - "build": "cd packages/telemetry && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", - "build:offline": "cd packages/telemetry && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", + "build": "cd packages/telemetry && npm run build && cd ../schema && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", + "build:offline": "cd packages/telemetry && npm run build && cd ../schema && npm run build && cd ../chord && npm run build && cd ../tui && npm run build && cd ../codemode && npm run build && cd ../mcp && npm run build && cd ../ai && npm run build:offline && cd ../durable && npm run build && cd ../env && npm run build && cd ../agent && npm run build && cd ../coding-agent && npm run build", "build:native:darwin": "npm --prefix packages/tui run build:native:darwin", "build:native:linux": "npm --prefix packages/tui run build:native:linux", "build:native:win32": "npm --prefix packages/tui run build:native:win32", - "check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && npm run check:browser-smoke", + "check": "node scripts/biome-check.mjs --write && npm run check:pinned-deps && npm run check:runtime-deps && npm run check:ts-imports && npm run check:entry-graphs && npm run check:install-lock:coding-agent && tsc --noEmit && tsc --noEmit -p packages/coding-agent/examples/tsconfig.json && npm run check:browser-smoke", "check:browser-smoke": "node scripts/check-browser-smoke.mjs", "check:pinned-deps": "node scripts/check-pinned-deps.mjs", "check:runtime-deps": "node scripts/check-runtime-deps.mjs", @@ -57,14 +56,12 @@ "prepare": "husky" }, "devDependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26", "@biomejs/biome": "2.3.5", "@biomejs/wasm-nodejs": "2.3.5", "@types/node": "22.19.19", "esbuild": "0.28.2", "husky": "9.1.7", "pnpm": "11.15.1", - "shx": "0.3.4", "typescript": "7.0.2", "yaml": "2.9.0" }, diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 2ae23784d..9e80e2abf 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Breaking Changes + +- `AgentTool` takes any JSON Schema object as `parameters` (`AgentTool`) and types `params` and `prepareArguments` through `@earendil-works/pi-schema` (`AgentToolParams`): pi-schema's builders give their static type, TypeBox schemas give TypeBox's static type for the common kinds (String, Number, Integer, Boolean, Null, Literal, Array, Object with optional and readonly properties, Union, Record, Unsafe, Unknown, Any; other kinds such as Enum, Tuple and Intersect give `unknown`), other schemas give `unknown`, `AgentTool` stays `any`. `outputSchema` is any JSON Schema object. `typebox` is no longer a dependency. Migration: build parameters with pi-schema (`obj({ path: str() })` instead of `Type.Object({ path: Type.String() })`), or keep TypeBox schemas (narrow `params` yourself only where they use other kinds). + ### Fixed - Fixed `streamProxy()` re-parsing the accumulated tool-call arguments on every delta and keeping their raw text on the partial content block ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/agent/README.md b/packages/agent/README.md index 38625ce11..a6fa8256b 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -455,18 +455,20 @@ const agent = new Agent({ ## Tools -Define tools using `AgentTool`: +Define tools using `AgentTool`. `parameters` is any JSON Schema object; built with `@earendil-works/pi-schema`'s builders, it also types `params` (other schemas give `unknown`, `AgentTool` gives `any`): ```typescript -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; -const readFileTool: AgentTool = { +const readFileParameters = obj({ + path: str({ description: "File path" }), +}); + +const readFileTool: AgentTool = { name: "read_file", label: "Read File", // For UI display description: "Read a file's contents", - parameters: Type.Object({ - path: Type.String({ description: "File path" }), - }), + parameters: readFileParameters, // Override execution mode for this tool (optional). // "sequential" forces the entire batch to run one at a time. // "parallel" allows concurrent execution with other tool calls. diff --git a/packages/agent/examples/mcp-codemode/tools.ts b/packages/agent/examples/mcp-codemode/tools.ts index 37e358c65..06d678670 100644 --- a/packages/agent/examples/mcp-codemode/tools.ts +++ b/packages/agent/examples/mcp-codemode/tools.ts @@ -18,7 +18,7 @@ import { renderDeclarations, } from "@earendil-works/pi-codemode"; import { type McpClient, toLlmContent } from "@earendil-works/pi-mcp"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; /** Provider tool names are limited to 64 characters of `[A-Za-z0-9_-]`. */ function toToolName(prefix: string, name: string): string { @@ -45,12 +45,12 @@ export async function createMcpTools(client: McpClient, prefix: string): Promise label: tool.title ?? tool.name, description: tool.description ?? tool.title ?? `MCP tool ${tool.name}`, // Providers require an object schema, and some reject one without `properties`. - parameters: Type.Unsafe({ + parameters: { ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {}, - }), - outputSchema: tool.outputSchema ? Type.Unsafe>(tool.outputSchema) : undefined, + }, + outputSchema: tool.outputSchema, execute: async (_toolCallId, params, signal, onUpdate) => { const result = await client.callTool(tool.name, params as Record, { signal, @@ -104,9 +104,11 @@ export function createNestedToolRunner( export const CODEMODE_TOOL_NAME = "codemode"; -const codemodeParameters = Type.Object({ - code: Type.String({ description: "JavaScript source. Top-level await and return work." }), -}); +const codemodeParameters = obj( + { code: str({ description: "JavaScript source. Top-level await and return work." }) }, + // Open, as the tool accepted extra arguments before it moved from TypeBox. + { open: true }, +); /** * A tool that runs model-written JavaScript in the codemode sandbox, where `tools.(args)` @@ -121,7 +123,6 @@ export function createCodemodeTool( const sandboxTools: CodemodeTool[] = callable.map((tool) => ({ name: tool.name, description: tool.description, - // TypeBox schemas are JSON Schema. inputSchema: tool.parameters as CodemodeJsonSchema, // What the script receives: the structured result if the tool declares one, else its text. outputSchema: (tool.outputSchema as CodemodeJsonSchema | undefined) ?? { type: "string" }, diff --git a/packages/agent/package.json b/packages/agent/package.json index d1231d95d..e00a8e1e7 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-agent-core", - "version": "1.4.0", + "version": "1.5.0", "description": "General-purpose agent with transport abstraction, state management, and attachment support", "type": "module", "main": "./dist/index.js", @@ -17,14 +17,14 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run build" }, "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "keywords": [ "ai", diff --git a/packages/agent/src/types.ts b/packages/agent/src/types.ts index dbfba74a0..24bdd7531 100644 --- a/packages/agent/src/types.ts +++ b/packages/agent/src/types.ts @@ -14,7 +14,7 @@ import type { TranscriptContext, Usage, } from "@earendil-works/pi-ai"; -import type { Static, TSchema } from "typebox"; +import type { Static } from "@earendil-works/pi-schema"; /** * Stream function used by the agent loop. `Models.streamSimple` satisfies @@ -462,27 +462,39 @@ export interface AgentToolCallOutcome { */ export type AgentToolUpdateCallback = (partialResult: AgentToolResult) => void; -/** Tool definition used by the agent runtime. */ -export interface AgentTool extends Tool { +/** + * The arguments type of a tool with parameter schema `TParameters`: pi-schema's `Static` (the builders' + * type, TypeBox's for the common TypeBox kinds, else `unknown`), and `any` for `AgentTool`. + */ +export type AgentToolParams = IsAny extends true ? any : Static; + +// `0 extends 1 & T` is resolved too early for a `T` constrained to `object`; this form waits for the argument. +type IsAny = boolean extends (T extends never ? true : false) ? true : false; + +/** + * Tool definition used by the agent runtime. `parameters` is any JSON Schema object; see {@link AgentToolParams} for + * the type `execute` receives. + */ +export interface AgentTool extends Tool { /** Human-readable label for UI display. */ label: string; /** * Optional compatibility shim for raw tool-call arguments before schema validation. * Must return an object that matches `TParameters`. */ - prepareArguments?: (args: unknown) => Static; + prepareArguments?: (args: unknown) => AgentToolParams; /** * JSON Schema of `structuredContent` in successful results. Tools that declare it should always * set `structuredContent`. */ - outputSchema?: TSchema; + outputSchema?: object; /** * Execute the tool call. Throw on failure, or return a result with `isError: true`; do not only * describe the failure in `content`. */ execute: ( toolCallId: string, - params: Static, + params: AgentToolParams, signal?: AbortSignal, onUpdate?: AgentToolUpdateCallback, ) => Promise>; diff --git a/packages/agent/test/agent-loop.test.ts b/packages/agent/test/agent-loop.test.ts index 4a9386a13..b3ea8c259 100644 --- a/packages/agent/test/agent-loop.test.ts +++ b/packages/agent/test/agent-loop.test.ts @@ -6,7 +6,7 @@ import { type ToolResultMessage, type UserMessage, } from "@earendil-works/pi-ai"; -import { Type } from "typebox"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { agentLoop, agentLoopContinue, runAgentLoop, runToolCall } from "../src/agent-loop.ts"; import { setDefaultStreamFn } from "../src/index.ts"; @@ -303,7 +303,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should handle tool calls and results", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const toolUsage = { input: 1, @@ -400,7 +400,7 @@ describe("agentLoop with AgentMessage", () => { // #10549 it("records how long execute() took on the tool result, excluding hooks", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const tool: AgentTool = { name: "echo", @@ -454,7 +454,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should not execute tool calls from a length-truncated assistant message", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -526,7 +526,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should execute mutated beforeToolCall args without revalidation", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: Array = []; const tool: AgentTool = { name: "echo", @@ -587,8 +587,8 @@ describe("agentLoop with AgentMessage", () => { }); it("should prepare tool arguments for validation", async () => { - const replaceSchema = Type.Object({ oldText: Type.String(), newText: Type.String() }); - const toolSchema = Type.Object({ edits: Type.Array(replaceSchema) }); + const replaceSchema = obj({ oldText: str(), newText: str() }); + const toolSchema = obj({ edits: arr(replaceSchema) }); const executed: Array> = []; const tool: AgentTool = { name: "edit", @@ -666,7 +666,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should emit tool_execution_end in completion order but persist tool results in source order", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -760,7 +760,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should inject queued messages after all tool calls complete", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -865,7 +865,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should force sequential execution when a tool has executionMode=sequential even with default parallel config", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -947,7 +947,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should force sequential execution when one of multiple tools has executionMode=sequential", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executionOrder: string[] = []; let releaseSlow: (() => void) | undefined; const slowDone = new Promise((resolve) => { @@ -1033,7 +1033,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should allow parallel execution when all tools have executionMode=parallel", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let firstResolved = false; let parallelObserved = false; let releaseFirst: (() => void) | undefined; @@ -1106,7 +1106,7 @@ describe("agentLoop with AgentMessage", () => { }); it("runs finishTurn after tool-result messages and before turn_end", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1215,7 +1215,7 @@ describe("agentLoop with AgentMessage", () => { ); it("action:end skips queue polling and next-turn preparation", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1305,7 +1305,7 @@ describe("agentLoop with AgentMessage", () => { }); it("lets a natural tool-result request satisfy continuation", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1501,7 +1501,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should use prepareNextTurn snapshot before continuing", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1578,7 +1578,7 @@ describe("agentLoop with AgentMessage", () => { }); it("picks up steering queued during prepareNextTurn before the next request", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -1631,7 +1631,7 @@ describe("agentLoop with AgentMessage", () => { }); it("action:end receives finalized turn context and stops before queue polling", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -1730,7 +1730,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should stop after a tool batch when every tool result sets terminate=true", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1781,7 +1781,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should stop after a blocked tool call when beforeToolCall sets terminate=true", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); let executed = false; const tool: AgentTool = { name: "echo", @@ -1839,7 +1839,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should continue after a mixed batch with one terminating blocked call", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const executed: string[] = []; const tool: AgentTool = { name: "echo", @@ -1897,7 +1897,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should continue after parallel tool calls when not all tool results terminate", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -1962,7 +1962,7 @@ describe("agentLoop with AgentMessage", () => { }); it("should allow afterToolCall to mark a tool batch as terminating", async () => { - const toolSchema = Type.Object({ value: Type.String() }); + const toolSchema = obj({ value: str() }); const tool: AgentTool = { name: "echo", label: "Echo", @@ -2131,13 +2131,13 @@ describe("agentLoopContinue with AgentMessage", () => { }); describe("runToolCall", () => { - const echoSchema = Type.Object({ value: Type.String() }); + const echoSchema = obj({ value: str() }); const echo: AgentTool = { name: "echo", label: "Echo", description: "Echo tool", parameters: echoSchema, - outputSchema: Type.Object({ value: Type.String() }), + outputSchema: obj({ value: str() }), async execute(_toolCallId, params, _signal, onUpdate) { onUpdate?.({ content: [{ type: "text", text: "partial" }], details: {} }); return { @@ -2151,7 +2151,7 @@ describe("runToolCall", () => { name: "failing", label: "Failing", description: "Returns an error result", - parameters: Type.Object({}), + parameters: obj({}), async execute() { return { content: [{ type: "text", text: "bad" }], details: { partial: true }, isError: true }; }, diff --git a/packages/agent/test/agent-tool-typebox.test.ts b/packages/agent/test/agent-tool-typebox.test.ts new file mode 100644 index 000000000..1c859b7ae --- /dev/null +++ b/packages/agent/test/agent-tool-typebox.test.ts @@ -0,0 +1,65 @@ +// AgentTool params typed from a TypeBox-built parameter schema. pi-agent-core does not depend on TypeBox: the schema +// below has TypeBox 1.x's type shapes as its d.mts declare them; pi-schema's own tests check the shapes against TypeBox. + +import { describe, expectTypeOf, it } from "vitest"; +import type { AgentTool } from "../src/types.ts"; + +interface TString { + "~kind": "String"; + type: "string"; +} + +interface TLiteral { + "~kind": "Literal"; + const: Value; +} + +interface TUnion { + "~kind": "Union"; + anyOf: Branches; +} + +interface TObject { + "~kind": "Object"; + type: "object"; + properties: Properties; + required: string[]; +} + +type TOptional = Schema & { "~optional": true }; + +const parameters: TObject<{ command: TString; mode: TOptional, TLiteral<"deep">]>> }> = { + "~kind": "Object", + type: "object", + properties: { + command: { "~kind": "String", type: "string" }, + mode: { + "~kind": "Union", + anyOf: [ + { "~kind": "Literal", const: "fast" }, + { "~kind": "Literal", const: "deep" }, + ], + "~optional": true, + }, + }, + required: ["command"], +}; + +type Params = { command: string; mode?: "fast" | "deep" }; + +describe("AgentTool params of a TypeBox schema", () => { + it("are typed by the schema in execute() and prepareArguments()", () => { + const tool: AgentTool = { + name: "bash", + label: "Bash", + description: "Runs a command", + parameters, + async execute(_toolCallId, params) { + expectTypeOf(params).toEqualTypeOf(); + return { content: [], details: undefined }; + }, + }; + expectTypeOf[1]>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + }); +}); diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 6fad6109f..9c8bddc39 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -7,7 +7,7 @@ import { toToolDeclaration, type UserMessage, } from "@earendil-works/pi-ai/compat"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { Agent, @@ -49,7 +49,7 @@ function createTool(name: string): AgentTool { name, label: name, description: `${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: name }], details: {} }), }; } @@ -146,7 +146,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -200,7 +200,7 @@ describe("Agent", () => { expect(update).toEqual({ role: "system", content: "", - toolsAdded: [{ name: "second", description: "second tool", parameters: Type.Object({}) }], + toolsAdded: [{ name: "second", description: "second tool", parameters: obj({}) }], toolsRemoved: [{ name: "first" }], timestamp: expect.any(Number), }); @@ -214,7 +214,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -239,7 +239,7 @@ describe("Agent", () => { role: "system", content: "", sections: { skills: "x" }, - toolsAdded: [{ name: "echo", description: "Echo input", parameters: Type.Object({}) }], + toolsAdded: [{ name: "echo", description: "Echo input", parameters: obj({}) }], timestamp: 1, }); }); @@ -284,7 +284,7 @@ describe("Agent", () => { name: "echo", label: "Echo", description: "Echo input", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ content: [{ type: "text", text: "echo" }], details: {} }), }; const agent = new Agent({ @@ -471,7 +471,7 @@ describe("Agent", () => { }); it("should ignore tool updates after the tool execution settles", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); let delayedUpdate: AgentToolUpdateCallback<{ status: string }> | undefined; const events: AgentEvent[] = []; const unhandledRejections: unknown[] = []; @@ -536,7 +536,7 @@ describe("Agent", () => { }); it("should ignore a settled parallel tool update while another tool is still running", async () => { - const toolSchema = Type.Object({}); + const toolSchema = obj({}); const slowStarted = createDeferred(); const settledToolEnded = createDeferred(); const releaseSlow = createDeferred(); @@ -858,7 +858,7 @@ describe("Agent", () => { }); it("keeps legacy prepareNextTurn signal callback behavior", async () => { - const schema = Type.Object({}); + const schema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", @@ -899,7 +899,7 @@ describe("Agent", () => { }); it("forwards finishTurn through AgentOptions with the active abort signal", async () => { - const schema = Type.Object({}); + const schema = obj({}); const tool: AgentTool = { name: "noop", label: "Noop", diff --git a/packages/agent/test/utils/calculate.ts b/packages/agent/test/utils/calculate.ts index 0a4fefc58..333cd5a0e 100644 --- a/packages/agent/test/utils/calculate.ts +++ b/packages/agent/test/utils/calculate.ts @@ -1,5 +1,5 @@ import type { Usage } from "@earendil-works/pi-ai"; -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import type { AgentTool, AgentToolResult } from "../../src/types.ts"; export interface CalculateResult extends AgentToolResult { @@ -16,9 +16,11 @@ export function calculate(expression: string): CalculateResult { } } -const calculateSchema = Type.Object({ - expression: Type.String({ description: "The mathematical expression to evaluate" }), -}); +const calculateSchema = obj( + { expression: str({ description: "The mathematical expression to evaluate" }) }, + // Open, as the tool accepted extra arguments before it moved from TypeBox. + { open: true }, +); type CalculateParams = Static; diff --git a/packages/agent/test/utils/get-current-time.ts b/packages/agent/test/utils/get-current-time.ts index a83a82af3..11996cc41 100644 --- a/packages/agent/test/utils/get-current-time.ts +++ b/packages/agent/test/utils/get-current-time.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { obj, opt, type Static, str } from "@earendil-works/pi-schema"; import type { AgentTool, AgentToolResult } from "../../src/types.ts"; export interface GetCurrentTimeResult extends AgentToolResult<{ utcTimestamp: number }> {} @@ -27,11 +27,11 @@ export async function getCurrentTime(timezone?: string): Promise; diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 996207fbd..b6e781ae5 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -2,8 +2,41 @@ ## [Unreleased] +### Fixed + +- Fixed OpenCode Go Claude Haiku 5.5 failing with a 400 (`Invalid signature in thinking block ... bound to a different conversation`) after the tool list, system prompt, or earlier history changed, which left the session failing on every retry. OpenCode Go and Zen Claude 5.5 models, Opus 5, and Fable 5.1 now use per-message effort and send `thinking.block_binding: drop_block`, like the Anthropic provider ([#10954](https://github.com/earendil-works/pi/issues/10954)) +- Fixed Anthropic non-strict tool schemas dropping root `$defs` and `definitions`, which left property `$ref`s pointing at nothing. Claude then produced wrong arguments for such tools (common in MCP servers), for example a JSON string instead of an object, and some Anthropic-compatible gateways rejected the request outright ([#9557](https://github.com/earendil-works/pi/issues/9557)) +- Fixed OpenRouter rejecting requests with a 400 (`This endpoint's maximum context length is ...`) for models whose output limit is close to the context window, such as DeepSeek flash. OpenRouter counts the input with its own tokenizer, which can be several percent higher than the provider's count, and pi left only 4,096 tokens spare. pi then treated the 400 as a context overflow and compacted the session, even at around 20% of the window. The default output limit now leaves 15% of the estimated input unused, and near a full window at most 32,768 output tokens are requested ([#10497](https://github.com/earendil-works/pi/issues/10497)) + +## [1.5.0] - 2026-11-02 + ### Breaking Changes +- The main entry no longer re-exports TypeBox (`Type`, `Static`, `TSchema`), and `typebox` is no longer a dependency. `Tool.parameters` is any JSON Schema object (`Tool`): `@earendil-works/pi-schema`'s builders, which also type the arguments through pi-schema's `Static`, TypeBox schemas, or plain objects. `validateToolCall()`/`validateToolArguments()` validate with pi-schema (`compile(schema, { coerce: true }).parse()`): every schema gets the same coercion, the superset of TypeBox's `Value.Convert` rules (which only applied to TypeBox-built schemas) and pi-ai's own, so plain JSON Schema tools (MCP tools) now also get, for example, `null` to `"null"` for a required string, `"3.5"` to 3 for an integer and a single value wrapped into an array. The error text lists pi-schema's messages (`- edits.0.oldText: missing`, `- limit: expected number, got string`) instead of TypeBox's. `StringEnum()` returns a pi-schema `Schema` with the same JSON (`{ type: "string", enum }`). The model and compat schemas (`providers/model-schema`, `providers/compat-schema`) are built with pi-schema, with the same JSON Schema, and their types come from pi-schema's `Static`; `ProviderCompat` is exported. Migration: + + ```ts + // Before + import { Type, type Static, StringEnum } from "@earendil-works/pi-ai"; + const params = Type.Object({ path: Type.String(), mode: StringEnum(["a", "b"]) }); + type Params = Static; + + // After: pi-schema builders (closed objects; pass { open: true } to allow other properties) + import { StringEnum } from "@earendil-works/pi-ai"; + import { obj, type Static, str } from "@earendil-works/pi-schema"; + const params = obj({ path: str(), mode: StringEnum(["a", "b"]) }); + type Params = Static; + + // Or keep TypeBox, imported from typebox itself (its schemas are JSON Schema) + import { Type, type Static } from "typebox"; + ``` +- Tool arguments are validated as JSON Schema only; nothing runs through TypeBox's own validator any more, so TypeBox features outside JSON Schema no longer apply to tool arguments: + - `Type.Refine()` refinements are not run: arguments they would reject are accepted. + - `Type.BigInt()`, `Type.Undefined()` and `Type.Void()` reject every JSON argument (TypeBox converted to them). + - Codecs (`Type.Codec()`) do not decode: the tool receives the encoded JSON value. + - Formats registered with TypeBox's `Format.Set()` are not checked; the standard formats plus `url` and `json-pointer-uri-fragment` are. + - Property names are names, not regular expressions: `"a.b"` no longer also converts the argument `axb`. + - `additionalProperties` converts only undeclared properties; TypeBox also converted declared ones with it when an object had more than one property. +- A `$ref` that names nothing (a missing definition, an external document) rejects only arguments at its place (`- t: unresolved reference #/definitions/Missing`) instead of failing every call of the tool, as TypeBox's rejection did before. - `appendGrammarToolInputJsonDelta()` (`api/constrained-sampling`) takes the input delta instead of the whole input so far and no longer closes the input; close it with the new `closeGrammarToolInputJson()` ([#9265](https://github.com/earendil-works/pi/issues/9265)) ### Added @@ -11,6 +44,8 @@ - Added optional `openBrowserOnConfirm` and `copyCodeOnConfirm` to `device_code` auth events and `OAuthDeviceCodeInfo`. Both default to `true`; set them to `false` to stop clients from offering to open the verification page or copy the code ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) - Added `StreamingJsonParser`, an incremental, tolerant parser for JSON text that arrives in pieces (each piece is read once; snapshots of the partial value never change once returned), and `ToolCallArgumentsStream`, which every provider now uses for streamed tool-call arguments; its `append()` also returns the changes each piece made (`JsonChange`: a value set at a path, or text appended to a string), for writers that keep their own copy ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Added GPT-6.1 Terra to the OpenAI, Azure OpenAI Responses, and OpenAI Codex providers with official pricing ($1 input, $0.05 cached input, $1.25 cache writes, $5 output, long-context tier above 272k), tool search, and mid-conversation effort changes. Unlike GPT-6.1 Sol it supports `off` (reasoning effort `none`); Codex serves it with a 500K context window +- Added `Usage.cost.totalFromProvider` to distinguish provider-reported billed totals from catalog estimates. Missing or `false` means an estimate; combined usage is marked provider-reported only when every part is ([#11178](https://github.com/earendil-works/pi/pull/11178)) +- Added `responseModel` to pi-messages terminal events, preserving the concrete model reported by providers such as Radius when a virtual model is requested ([#11265](https://github.com/earendil-works/pi/pull/11265)) ### Changed @@ -32,6 +67,12 @@ - Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from another model that called tools. On the OpenAI and OpenCode providers they now declare their tools in the top-level tool list instead of loading them through `additional_tools` or tool search ([#11415](https://github.com/earendil-works/pi/issues/11415)) - Fixed Bedrock requests failing when `strict: "prefer"` tools exceed Claude's strict tool limits, such as more than 16 optional parameters across all strict tools; requests now retry once with preferred tools non-strict while keeping `strict: "require"` tools strict, as the Anthropic provider already does ([#11416](https://github.com/earendil-works/pi/issues/11416)) - Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing Claude's earlier thinking after each tool call. They used the Chat Completions API, where pi dropped the thinking signature, and OpenRouter then ignored the unsigned thinking. They now use the Anthropic Messages API with adaptive thinking, like the versioned Claude models ([#9534](https://github.com/earendil-works/pi/issues/9534)) +- Fixed OpenRouter cost totals using catalog estimates instead of reported billing for Chat Completions and Anthropic Messages. BYOK totals include both OpenRouter charges and reported upstream costs; component costs remain catalog estimates ([#9980](https://github.com/earendil-works/pi/issues/9980)) +- Fixed OpenCode Zen and OpenCode Go requests using shared endpoints instead of their protocol-specific OpenAI, Anthropic, Google, and System One endpoints ([#11434](https://github.com/earendil-works/pi/pull/11434) by [@vimtor](https://github.com/vimtor)) +- Fixed Zhipu/Z.ai usage-limit errors (code `1308`) being treated as transient HTTP 429 errors, causing automatic retries that cannot succeed until the usage cap resets ([#11537](https://github.com/earendil-works/pi/issues/11537)) +- Fixed LiteLLM's `ContextWindowExceededError` not being recognized as context overflow ([#11492](https://github.com/earendil-works/pi/issues/11492)) +- Fixed Gemini tool schemas sending unsupported `const` keywords. They are now converted to single-value `enum` schemas without rewriting literal data in defaults, examples, or enum values, or losing properties named `__proto__` ([#11308](https://github.com/earendil-works/pi/issues/11308)) +- Fixed the faux provider counting simulated cache-write tokens as both input and cache writes ## [1.4.0] - 2026-10-21 diff --git a/packages/ai/README.md b/packages/ai/README.md index ac4275a21..d7a7a21e8 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -103,15 +103,16 @@ Unified LLM API with provider collections, automatic auth resolution, token and npm install @earendil-works/pi-ai ``` -TypeBox exports are re-exported from `@earendil-works/pi-ai`: `Type`, `Static`, and `TSchema`. +Tool parameters are JSON Schema. The examples build them with `@earendil-works/pi-schema`, which also types the arguments (`npm install @earendil-works/pi-schema`); TypeBox schemas and plain JSON Schema objects work as well. ## Quick Start You build a `Models` collection of providers and stream through it. The quickest start registers every built-in provider; apps that care about bundle size register individual providers instead (see [Provider Factories](#provider-factories) and [Bundling and Tree Shaking](#bundling-and-tree-shaking)). ```typescript -import { Type, type Context, type Tool } from '@earendil-works/pi-ai'; +import { type Context, type Tool } from '@earendil-works/pi-ai'; import { builtinModels } from '@earendil-works/pi-ai/providers/all'; +import { obj, opt, str } from '@earendil-works/pi-schema'; // A Models collection with every built-in provider registered const models = builtinModels(); @@ -119,12 +120,12 @@ const models = builtinModels(); // Sync lookup against the collection const model = models.getModel('openai', 'gpt-4o-mini')!; -// Define tools with TypeBox schemas for type safety and validation +// Define tools with JSON Schema; pi-schema's builders also type the arguments const tools: Tool[] = [{ name: 'get_time', description: 'Get the current time', - parameters: Type.Object({ - timezone: Type.Optional(Type.String({ description: 'Optional timezone (e.g., America/New_York)' })) + parameters: obj({ + timezone: opt(str({ description: 'Optional timezone (e.g., America/New_York)' })) }) }]; @@ -538,34 +539,35 @@ Amazon Bedrock resolves ambient AWS credentials (`AWS_PROFILE`, access key pairs ## Tools -Tools enable LLMs to interact with external systems. This library uses TypeBox schemas for type-safe tool definitions with automatic validation using TypeBox's built-in validator and value conversion utilities. TypeBox schemas can be serialized and deserialized as plain JSON, making them ideal for distributed systems. +Tools enable LLMs to interact with external systems. A tool's `parameters` is a JSON Schema object: built with `@earendil-works/pi-schema`'s builders (which also type the arguments through its `Static`), with TypeBox, or written by hand. `validateToolCall()` and `validateToolArguments()` validate with pi-schema, coercing loosely typed arguments first (numeric strings, `"true"`, a single value for an array, `null` for an omitted optional property). Schemas are plain JSON, so they serialize with the rest of a context. ### Defining Tools ```typescript -import { Type, type Tool, StringEnum } from '@earendil-works/pi-ai'; +import { type Tool, StringEnum } from '@earendil-works/pi-ai'; +import { arr, obj, str } from '@earendil-works/pi-schema'; -// Define tool parameters with TypeBox +// Define tool parameters with pi-schema's builders const weatherTool: Tool = { name: 'get_weather', description: 'Get current weather for a location', - parameters: Type.Object({ - location: Type.String({ description: 'City name or coordinates' }), + parameters: obj({ + location: str({ description: 'City name or coordinates' }), units: StringEnum(['celsius', 'fahrenheit'], { default: 'celsius' }) }) }; -// Note: For Google API compatibility, use StringEnum helper instead of Type.Enum -// Type.Enum generates anyOf/const patterns that Google doesn't support +// Note: StringEnum gives { type: "string", enum: [...] }, which every provider accepts, +// including Google's, which rejects anyOf/const patterns const bookMeetingTool: Tool = { name: 'book_meeting', description: 'Schedule a meeting', - parameters: Type.Object({ - title: Type.String({ minLength: 1 }), - startTime: Type.String({ format: 'date-time' }), - endTime: Type.String({ format: 'date-time' }), - attendees: Type.Array(Type.String({ format: 'email' }), { minItems: 1 }) + parameters: obj({ + title: str({ minLength: 1 }), + startTime: str({ format: 'date-time' }), + endTime: str({ format: 'date-time' }), + attendees: arr(str({ format: 'email' }), { minItems: 1 }) }) }; ``` @@ -578,10 +580,7 @@ Tools can opt in to provider-side constrained sampling. For JSON-schema tools, ` const strictTool: Tool = { name: 'edit_file', description: 'Edit a file', - parameters: Type.Object({ - path: Type.String(), - content: Type.String() - }, { additionalProperties: false }), + parameters: obj({ path: str(), content: str() }), // obj() is closed: additionalProperties: false constrainedSampling: { type: 'json_schema', strict: 'prefer' } }; ``` @@ -592,9 +591,7 @@ Strict JSON-schema constrained sampling is supported for OpenAI, Anthropic, supp const patchTool: Tool = { name: 'apply_patch', description: 'Apply a patch', - parameters: Type.Object({ - input: Type.String() - }, { additionalProperties: false }), + parameters: obj({ input: str() }), constrainedSampling: { type: 'grammar', variants: { @@ -698,7 +695,7 @@ for await (const event of s) { ### Validating Tool Arguments -When implementing your own tool execution loop, use `validateToolCall` to validate arguments before passing them to your tools: +When implementing your own tool execution loop, use `validateToolCall` to validate arguments before passing them to your tools. It coerces loosely typed arguments with pi-schema (`compile(schema, { coerce: true }).parse()`), returns the coerced arguments, and throws an error listing pi-schema's errors (`path: message`) followed by the arguments it received: ```typescript import { validateToolCall, type Tool } from '@earendil-works/pi-ai'; @@ -894,7 +891,7 @@ console.log(model.output); // ['image'] or ['image', 'text'] ## Classification -Classifier models consume structured JSON state, and optionally images, and answer one or more typed questions. They do not use chat or image-generation APIs. TypeSafe's Jev model, Cloudflare's Clef models, and OpenAI's GPT-6 Luna are available from these built-in providers: +Classifier models consume structured JSON state, and optionally images, and answer one or more typed questions. They do not use chat or image-generation APIs. TypeSafe's Jev model, Cloudflare's Clef models, OpenAI's GPT-6 Luna, and Microsoft-Decision-1 are available from these built-in providers: | Provider | Model IDs | Auth | | --- | --- | --- | @@ -904,6 +901,7 @@ Classifier models consume structured JSON state, and optionally images, and answ | `vercel-ai-gateway` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY` | | `opencode` | `jev-1.13`, `jev-1.13-free` | `OPENCODE_API_KEY` | | `openai` | `gpt-6-luna` | `OPENAI_API_KEY` | +| `github-copilot` | `microsoft-decision-1` | `COPILOT_GITHUB_TOKEN` or GitHub Copilot OAuth | ```typescript import { builtinModels } from '@earendil-works/pi-ai/providers/all'; @@ -1728,8 +1726,8 @@ models.setProvider(openaiProvider()); Rules: -- `@earendil-works/pi-ai/models` exports the model runtime (`createModels`, `createProvider`, model helpers, and their types) without TypeBox, built-in catalogs, or SDK implementations. Other types can still use `import type` from the root. -- `@earendil-works/pi-ai` is the core entrypoint and does not import built-in catalogs, real provider factories, or SDK implementations, but it eagerly imports TypeBox and schema validation. Unbundled Node scripts do not tree-shake its unused exports; prefer `./models`, `./providers/faux`, and specific `./utils/*` subpaths when those are all you need. +- `@earendil-works/pi-ai/models` exports the model runtime (`createModels`, `createProvider`, model helpers, and their types) without schema validation, built-in catalogs, or SDK implementations. Other types can still use `import type` from the root. +- `@earendil-works/pi-ai` is the core entrypoint and does not import built-in catalogs, real provider factories, or SDK implementations, but it eagerly imports schema validation (pi-schema). Unbundled Node scripts do not tree-shake its unused exports; prefer `./models`, `./providers/faux`, and specific `./utils/*` subpaths when those are all you need. - `@earendil-works/pi-ai/providers/` imports that provider's catalog and lazy API wrapper only. - `@earendil-works/pi-ai/providers/all` imports every built-in provider factory and all catalogs. Use it only when you want the full built-in set. - With code splitting, provider SDKs stay in lazy chunks and load on first request. diff --git a/packages/ai/catalog/providers/github-copilot.ts b/packages/ai/catalog/providers/github-copilot.ts index 2bea3586b..b91a03213 100644 --- a/packages/ai/catalog/providers/github-copilot.ts +++ b/packages/ai/catalog/providers/github-copilot.ts @@ -1,4 +1,4 @@ -import type { Api, Model, ThinkingLevelMap } from "../../src/types.ts"; +import type { Api, ClassifierModel, Model, ThinkingLevelMap } from "../../src/types.ts"; import { addMissing, type ChatModel, defineProvider, mergeCompat, mergeThinking, modelsDevModels } from "../define.ts"; import { claude55ThinkingLevels, supportsClaudeMidConvoSystemMessages } from "../families/anthropic.ts"; import { @@ -67,6 +67,22 @@ const GPT_6_LUNA: Model<"openai-responses"> = { headers: { ...STATIC_HEADERS }, }; +// The authenticated Copilot catalog lists this decision model on /v1/systemone (checked 2026-11-02) without +// a context window or price, and GitHub's Copilot pricing page does not list it. These are Azure's values, +// which OpenRouter and Vercel AI Gateway also list. +const MICROSOFT_DECISION_1: ClassifierModel<"typesafe-system-one"> = { + type: "classifier", + id: "microsoft-decision-1", + name: "Microsoft-Decision-1", + api: "typesafe-system-one", + provider: "github-copilot", + baseUrl: BASE_URL, + input: ["text"], + cost: { input: 0.042, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 32768, + headers: { ...STATIC_HEADERS }, +}; + export default defineProvider({ id: "github-copilot", models: (ctx) => @@ -94,6 +110,7 @@ export default defineProvider({ }), [GPT_6_LUNA], ).map((model) => (EXTENDED_CONTEXT_MODEL_IDS.has(model.id) ? { ...model, contextWindow: 1000000 } : model)), + classifiers: () => [MICROSOFT_DECISION_1], compat: { "openai-completions": { supportsStrictMode: true } }, finalize: (model) => { if (model.id.startsWith("gpt-5")) mergeThinking(model, { minimal: "low" }); diff --git a/packages/ai/catalog/providers/opencode.ts b/packages/ai/catalog/providers/opencode.ts index 36057c764..db981e4c3 100644 --- a/packages/ai/catalog/providers/opencode.ts +++ b/packages/ai/catalog/providers/opencode.ts @@ -7,7 +7,11 @@ import { modelsDevModels, type ProviderContext, } from "../define.ts"; -import { supportsClaudeMidConvoSystemMessages } from "../families/anthropic.ts"; +import { + enableClaudeMidConvoEffort, + supportsClaudeMidConvoEffort, + supportsClaudeMidConvoSystemMessages, +} from "../families/anthropic.ts"; import { googleThinkingLevels } from "../families/google.ts"; import { isKimiK3FamilyId } from "../families/kimi.ts"; import { enableOpenAiGrammarTools, enableOpenAiMidConvoSystemMessages } from "../families/openai.ts"; @@ -162,6 +166,9 @@ export function finalizeOpencodeModel(model: ChatModel): void { if (model.provider === "opencode" && supportsClaudeMidConvoSystemMessages(model.id)) { mergeCompat(messagesModel, { supportsMidConvoSystemMessages: true }); } + // Zen and Go forward per-message effort and `thinking.block_binding`. Without + // `drop_block`, Go rejects replayed thinking after the system prompt or tools change. + if (supportsClaudeMidConvoEffort(model.id)) enableClaudeMidConvoEffort(messagesModel); // OpenCode Qwen 3.8 Flash emits and accepts thinking blocks with empty signatures. if (model.id === "qwen3.8-flash") mergeCompat(messagesModel, { allowEmptySignature: true }); } diff --git a/packages/ai/package.json b/packages/ai/package.json index 6e04980f0..57e8f262c 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-ai", - "version": "1.4.0", + "version": "1.5.0", "description": "Unified LLM API with automatic model discovery and provider configuration", "type": "module", "main": "./dist/index.js", @@ -56,27 +56,27 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "generate-models": "node catalog/generate.ts --strict", "hydrate-model-data": "node catalog/generate.ts --strict --data-only", "generate-model-catalog": "node catalog/generate.ts --strict --json-only --json-output ../../.artifacts/model-catalog", "check:model-data": "node scripts/check-model-data.ts", "build": "npm run generate-models && npm run build:offline", - "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && shx rm -rf dist/providers/data && shx cp -r src/providers/data dist/providers/data", + "build:offline": "npm run check:model-data && tsc -p tsconfig.build.json && node ../../scripts/package-files.mjs clean dist/providers/data && node ../../scripts/package-files.mjs copy src/providers/data dist/providers", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "keywords": [ "ai", @@ -101,6 +101,7 @@ "devDependencies": { "@types/node": "22.19.19", "canvas": "3.2.3", + "typebox": "1.3.27", "vitest": "4.1.12" } } diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index dd827254e..a6fd502bd 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -1896,11 +1896,20 @@ function convertTools( ? undefined : resolveJsonSchemaStrictSampling(tool, supportsStrictTools, isAnthropicStrictUnsupportedKeyword); const parameters = getJsonSchemaToolParameters(tool, strict); - const schema = parameters as { properties?: unknown; required?: string[] }; + const schema = parameters as { + properties?: unknown; + required?: string[]; + $defs?: unknown; + definitions?: unknown; + }; + // Keep shared definitions so property $refs still resolve. Other root keywords stay out: + // Anthropic rejects root anyOf/oneOf/allOf with a 400. const legacyInputSchema = { type: "object" as const, properties: schema.properties ?? {}, required: schema.required ?? [], + ...(schema.$defs === undefined ? {} : { $defs: schema.$defs }), + ...(schema.definitions === undefined ? {} : { definitions: schema.definitions }), }; const inputSchema = strict === true diff --git a/packages/ai/src/api/simple-options.ts b/packages/ai/src/api/simple-options.ts index 1134c2a88..498410f9f 100644 --- a/packages/ai/src/api/simple-options.ts +++ b/packages/ai/src/api/simple-options.ts @@ -13,12 +13,25 @@ import type { import { estimateContextTokens } from "../utils/estimate.ts"; const CONTEXT_SAFETY_TOKENS = 4096; +// OpenRouter counts the input with its own tokenizer and rejects input + max_tokens above the +// window before routing. It counted prose up to 12% above the provider's count (#10497). +const CONTEXT_SAFETY_RATIO = 0.15; +// The 15% margin is kept only while it leaves at least this much output. Otherwise (a nearly full +// large window, or any small window such as a 32k local model) only the fixed margin is kept, and +// output is capped at this size so an OpenRouter overcount still fits for a while before compaction. +const MIN_OUTPUT_WITH_RATIO_MARGIN = 32768; const MIN_MAX_TOKENS = 1; +/** Fit maxTokens into the context window, leaving room for the input being counted higher than estimated. */ export function clampMaxTokensToContext(model: Model, context: TranscriptContext, maxTokens: number): number { if (model.contextWindow <= 0) return Math.max(MIN_MAX_TOKENS, maxTokens); - const available = model.contextWindow - estimateContextTokens(context).tokens - CONTEXT_SAFETY_TOKENS; - return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, available)); + const input = estimateContextTokens(context).tokens; + const margin = Math.max(CONTEXT_SAFETY_TOKENS, Math.ceil(input * CONTEXT_SAFETY_RATIO)); + let room = model.contextWindow - input - margin; + if (room < MIN_OUTPUT_WITH_RATIO_MARGIN) { + room = Math.min(MIN_OUTPUT_WITH_RATIO_MARGIN, model.contextWindow - input - CONTEXT_SAFETY_TOKENS); + } + return Math.min(maxTokens, Math.max(MIN_MAX_TOKENS, room)); } export function resolveSamplingParams( diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 54a0345f1..0c9197ae6 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -1,6 +1,3 @@ -export type { Static, TSchema } from "typebox"; -export { Type } from "typebox"; - // Core only, side-effect free: no generated catalogs, no provider factories, // no api-registry, no OAuth implementations, no compat. Provider factories // live under "@earendil-works/pi-ai/providers/*", API implementations under @@ -42,9 +39,9 @@ export * from "./utils/json-parse.ts"; export * from "./utils/overflow.ts"; export * from "./utils/retry.ts"; export * from "./utils/streaming-json.ts"; +export * from "./utils/string-enum.ts"; export { contentText, getSystemMessageText, renderSystemMessageUpdate } from "./utils/text.ts"; export * from "./utils/transcript.ts"; -export * from "./utils/typebox-helpers.ts"; export { uuidv7 } from "./utils/uuid.ts"; export * from "./utils/validation.ts"; export { PI_AI_VERSION } from "./utils/version.ts"; diff --git a/packages/ai/src/providers/compat-schema.ts b/packages/ai/src/providers/compat-schema.ts index f7e485159..792b7a1b7 100644 --- a/packages/ai/src/providers/compat-schema.ts +++ b/packages/ai/src/providers/compat-schema.ts @@ -1,17 +1,35 @@ -import { type Static, type TProperties, type TSchemaOptions, Type } from "typebox"; +import { + type Annotations, + arr, + bool, + nul, + num, + opt, + type Properties, + type Static, + str, + union, +} from "@earendil-works/pi-schema"; import { ModelCostSchema } from "./model-schema.ts"; +import { openObject, stringLiteral, stringRecord } from "./schema-builders.ts"; -function sessionAffinityFormat(options?: TSchemaOptions) { - return Type.Union([Type.Literal("openai"), Type.Literal("openai-nosession"), Type.Literal("openrouter")], options); +function sessionAffinityFormat(options?: Annotations<"openai" | "openai-nosession" | "openrouter">) { + return union([stringLiteral("openai"), stringLiteral("openai-nosession"), stringLiteral("openrouter")], options); } export const SessionAffinityFormatSchema = sessionAffinityFormat({ description: "Session-affinity header format used to route related requests consistently.", }); -function thinkingTokenBudgetField(options?: TSchemaOptions) { - return Type.Union( - [Type.Literal("thinking_token_budget"), Type.Literal("thinking_budget"), Type.Literal("thinking_budget_tokens")], +function thinkingTokenBudgetField( + options?: Annotations<"thinking_token_budget" | "thinking_budget" | "thinking_budget_tokens">, +) { + return union( + [ + stringLiteral("thinking_token_budget"), + stringLiteral("thinking_budget"), + stringLiteral("thinking_budget_tokens"), + ], options, ); } @@ -21,84 +39,78 @@ export const ThinkingTokenBudgetFieldSchema = thinkingTokenBudgetField({ 'Top-level request field name used by OpenAI-compatible endpoints to cap reasoning tokens. "thinking_token_budget" is used by vLLM, "thinking_budget" by Qwen, DashScope, or SGLang, and "thinking_budget_tokens" by llama.cpp.', }); -export const ChatTemplateKwargValueSchema = Type.Union([ - Type.String(), - Type.Number(), - Type.Boolean(), - Type.Null(), - Type.Object({ - $var: Type.Union([ - Type.Literal("thinking.enabled"), - Type.Literal("thinking.effort"), - Type.Literal("thinking.budget"), +export const ChatTemplateKwargValueSchema = union([ + str(), + num(), + bool(), + nul(), + openObject({ + $var: union([ + stringLiteral("thinking.enabled"), + stringLiteral("thinking.effort"), + stringLiteral("thinking.budget"), ]), - omitWhenOff: Type.Optional(Type.Boolean()), + omitWhenOff: opt(bool()), }), ]); function percentileCutoffs(metric: string) { - return Type.Object({ - p50: Type.Optional(Type.Number({ description: `${metric} at the 50th percentile.` })), - p75: Type.Optional(Type.Number({ description: `${metric} at the 75th percentile.` })), - p90: Type.Optional(Type.Number({ description: `${metric} at the 90th percentile.` })), - p99: Type.Optional(Type.Number({ description: `${metric} at the 99th percentile.` })), + return openObject({ + p50: opt(num({ description: `${metric} at the 50th percentile.` })), + p75: opt(num({ description: `${metric} at the 75th percentile.` })), + p90: opt(num({ description: `${metric} at the 90th percentile.` })), + p99: opt(num({ description: `${metric} at the 99th percentile.` })), }); } -export const OpenRouterRoutingSchema = Type.Object( +export const OpenRouterRoutingSchema = openObject( { - allow_fallbacks: Type.Optional( - Type.Boolean({ description: "Whether to allow backup providers to serve requests.", default: true }), + allow_fallbacks: opt( + bool({ description: "Whether to allow backup providers to serve requests.", default: true }), ), - require_parameters: Type.Optional( - Type.Boolean({ + require_parameters: opt( + bool({ description: "Whether to filter providers to only those that support all parameters in the request.", default: false, }), ), - data_collection: Type.Optional( - Type.Union([Type.Literal("deny"), Type.Literal("allow")], { + data_collection: opt( + union([stringLiteral("deny"), stringLiteral("allow")], { description: 'Data collection setting. "allow": allow providers that may store or train on data. "deny": only use providers that do not collect user data.', default: "allow", }), ), - zdr: Type.Optional( - Type.Boolean({ description: "Whether to restrict routing to only ZDR (Zero Data Retention) endpoints." }), + zdr: opt(bool({ description: "Whether to restrict routing to only ZDR (Zero Data Retention) endpoints." })), + enforce_distillable_text: opt( + bool({ description: "Whether to restrict routing to only models that allow text distillation." }), ), - enforce_distillable_text: Type.Optional( - Type.Boolean({ description: "Whether to restrict routing to only models that allow text distillation." }), - ), - order: Type.Optional( - Type.Array(Type.String(), { + order: opt( + arr(str(), { description: "An ordered list of provider names or slugs to try in sequence, falling back to the next if unavailable.", }), ), - only: Type.Optional( - Type.Array(Type.String(), { + only: opt( + arr(str(), { description: "List of provider names or slugs to exclusively allow for this request.", }), ), - ignore: Type.Optional( - Type.Array(Type.String(), { description: "List of provider names or slugs to skip for this request." }), - ), - quantizations: Type.Optional( - Type.Array(Type.String(), { + ignore: opt(arr(str(), { description: "List of provider names or slugs to skip for this request." })), + quantizations: opt( + arr(str(), { description: 'A list of quantization levels to filter providers by, for example ["fp16", "bf16", "fp8", "fp6", "int8", "int4", "fp4", "fp32"].', }), ), - sort: Type.Optional( - Type.Union( + sort: opt( + union( [ - Type.String(), - Type.Object({ - by: Type.Optional( - Type.String({ description: 'The sorting metric, such as "price", "throughput", or "latency".' }), - ), - partition: Type.Optional( - Type.Union([Type.String(), Type.Null()], { + str(), + openObject({ + by: opt(str({ description: 'The sorting metric, such as "price", "throughput", or "latency".' })), + partition: opt( + union([str(), nul()], { description: 'Partitioning strategy: "model" or "none".', default: "model", }), @@ -111,33 +123,25 @@ export const OpenRouterRoutingSchema = Type.Object( }, ), ), - max_price: Type.Optional( - Type.Object( + max_price: opt( + openObject( { - prompt: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per million prompt tokens." }), - ), - completion: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per million completion tokens." }), - ), - image: Type.Optional(Type.Union([Type.Number(), Type.String()], { description: "Price per image." })), - audio: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per audio unit." }), - ), - request: Type.Optional( - Type.Union([Type.Number(), Type.String()], { description: "Price per request." }), - ), + prompt: opt(union([num(), str()], { description: "Price per million prompt tokens." })), + completion: opt(union([num(), str()], { description: "Price per million completion tokens." })), + image: opt(union([num(), str()], { description: "Price per image." })), + audio: opt(union([num(), str()], { description: "Price per audio unit." })), + request: opt(union([num(), str()], { description: "Price per request." })), }, { description: "Maximum price per million tokens in USD." }, ), ), - preferred_min_throughput: Type.Optional( - Type.Union([Type.Number(), percentileCutoffs("Minimum tokens per second")], { + preferred_min_throughput: opt( + union([num(), percentileCutoffs("Minimum tokens per second")], { description: "Preferred minimum throughput in tokens per second. A number applies to p50.", }), ), - preferred_max_latency: Type.Optional( - Type.Union([Type.Number(), percentileCutoffs("Maximum latency in seconds")], { + preferred_max_latency: opt( + union([num(), percentileCutoffs("Maximum latency in seconds")], { description: "Preferred maximum latency in seconds. A number applies to p50.", }), ), @@ -148,16 +152,16 @@ export const OpenRouterRoutingSchema = Type.Object( }, ); -export const VercelGatewayRoutingSchema = Type.Object( +export const VercelGatewayRoutingSchema = openObject( { - only: Type.Optional( - Type.Array(Type.String(), { + only: opt( + arr(str(), { description: 'List of provider slugs to exclusively use for this request, for example ["bedrock", "anthropic"].', }), ), - order: Type.Optional( - Type.Array(Type.String(), { + order: opt( + arr(str(), { description: 'List of provider slugs to try in order, for example ["anthropic", "openai"].', }), ), @@ -168,378 +172,386 @@ export const VercelGatewayRoutingSchema = Type.Object( }, ); -export const AnthropicAllowedFallbackModelSchema = Type.Object( +export const AnthropicAllowedFallbackModelSchema = openObject( { - provider: Type.String({ minLength: 1 }), - model: Type.String({ minLength: 1 }), + provider: str({ minLength: 1 }), + model: str({ minLength: 1 }), cost: ModelCostSchema, }, { description: "An Anthropic server-side refusal fallback model with local pricing metadata." }, ); -function optionalCompatBoolean(options?: TSchemaOptions) { - return Type.Optional(Type.Boolean(options)); +function optionalCompatBoolean(options?: Annotations) { + return opt(bool(options)); } -function optionalSessionAffinityFormat(options?: TSchemaOptions) { - return Type.Optional(sessionAffinityFormat(options)); +function optionalSessionAffinityFormat(options?: Annotations<"openai" | "openai-nosession" | "openrouter">) { + return opt(sessionAffinityFormat(options)); } -export const OpenAICompletionsCompatSchema = Type.Object( - { - supportsStore: Type.Optional( - Type.Boolean({ - description: "Whether the provider supports the store field. Default: auto-detected from URL.", - }), - ), - supportsDeveloperRole: optionalCompatBoolean({ - description: - "Whether the provider supports the developer role instead of system. Default: auto-detected from URL.", +const OpenAICompletionsCompatProperties = { + supportsStore: opt( + bool({ + description: "Whether the provider supports the store field. Default: auto-detected from URL.", }), - supportsReasoningEffort: Type.Optional( - Type.Boolean({ - description: "Whether the provider supports reasoning_effort. Default: auto-detected from URL.", - }), - ), - supportsUsageInStreaming: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports stream_options.include_usage for token usage in streaming responses.", - default: true, - }), - ), - supportsFinishReason: Type.Optional( - Type.Boolean({ - description: - "Whether streamed responses include finish_reason. When false, pi infers stop or toolUse when the stream ends.", - default: true, - }), - ), - maxTokensField: Type.Optional( - Type.Union([Type.Literal("max_completion_tokens"), Type.Literal("max_tokens")], { - description: "Which field to use for max tokens. Default: auto-detected from URL.", - }), - ), - requiresToolResultName: Type.Optional( - Type.Boolean({ description: "Whether tool results require the name field. Default: auto-detected from URL." }), - ), - requiresAssistantAfterToolResult: Type.Optional( - Type.Boolean({ - description: - "Whether a user message after tool results requires an assistant message in between. Default: auto-detected from URL.", - }), - ), - requiresThinkingAsText: Type.Optional( - Type.Boolean({ - description: - "Whether thinking blocks must be converted to text blocks with delimiters. Default: auto-detected from URL.", - }), - ), - requiresReasoningContentOnAssistantMessages: Type.Optional( - Type.Boolean({ - description: - "Whether all replayed assistant messages must include an empty reasoning_content field when reasoning is enabled. Default: auto-detected from URL.", - }), - ), - thinkingFormat: Type.Optional( - Type.Union( - [ - Type.Literal("openai"), - Type.Literal("openrouter"), - Type.Literal("deepseek"), - Type.Literal("together"), - Type.Literal("baseten"), - Type.Literal("zai"), - Type.Literal("qwen"), - Type.Literal("chat-template"), - Type.Literal("qwen-chat-template"), - Type.Literal("string-thinking"), - Type.Literal("ant-ling"), - ], - { - description: - 'Format for reasoning or thinking parameters. When omitted, Pi auto-detects the format from the provider URL. "openai" uses reasoning_effort, "openrouter" uses reasoning.effort, "deepseek" uses thinking.type plus reasoning_effort when supported, "together" uses reasoning.enabled plus reasoning_effort when supported, "baseten" uses configurable chat_template_args plus reasoning_effort when supported, "zai" uses thinking.type, "qwen" uses top-level enable_thinking, "qwen-chat-template" uses chat_template_kwargs.enable_thinking and preserve_thinking, "chat-template" uses configurable chat_template_kwargs, "string-thinking" uses top-level thinking, and "ant-ling" uses reasoning.effort only when the mapped effort is non-null.', - }, - ), - ), - chatTemplateKwargs: Type.Optional( - Type.Record(Type.String(), ChatTemplateKwargValueSchema, { - description: - 'Kwargs sent as chat_template_kwargs when thinkingFormat is "chat-template". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', - }), - ), - chatTemplateArgs: Type.Optional( - Type.Record(Type.String(), ChatTemplateKwargValueSchema, { - description: - 'Arguments sent as chat_template_args when thinkingFormat is "baseten". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', - }), - ), - openRouterRouting: Type.Optional(OpenRouterRoutingSchema), - vercelGatewayRouting: Type.Optional(VercelGatewayRoutingSchema), - zaiToolStream: Type.Optional( - Type.Boolean({ - description: "Whether z.ai supports top-level tool_stream for streaming tool call deltas.", - default: false, - }), - ), - thinkingTokenBudgetField: Type.Optional( - thinkingTokenBudgetField({ - description: - 'Top-level request field used to cap reasoning tokens from thinkingBudgets. Reasoning and the answer share max_tokens on these endpoints. "thinking_token_budget" is vLLM, "thinking_budget" is Qwen, DashScope, or SGLang, and "thinking_budget_tokens" is llama.cpp. Off by default and not set on the generated catalog.', - }), - ), - supportsThinkingTokenBudget: Type.Optional( - Type.Boolean({ - description: - 'Alias for thinkingTokenBudgetField: "thinking_token_budget" (vLLM). Prefer thinkingTokenBudgetField.', - default: false, - }), - ), - supportsOpenAIGrammarTools: optionalCompatBoolean({ - description: - "Whether the provider supports OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", - default: false, - }), - supportsMidConvoSystemMessages: optionalCompatBoolean({ - description: - "Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", - default: false, - }), - supportsMidConvoToolAdditions: Type.Optional( - Type.Boolean({ - description: - "Whether system messages can introduce additional tools mid-conversation. Requires supportsMidConvoSystemMessages. The generated catalog enables this for capable models.", - default: false, - }), - ), - supportsStrictMode: optionalCompatBoolean({ - description: - "Whether the provider supports the strict field in tool definitions. Generated capable models enable it explicitly.", - default: false, - }), - cacheControlFormat: Type.Optional( - Type.Literal("anthropic", { - description: - "Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.", - }), - ), - sendSessionAffinityHeaders: optionalCompatBoolean({ - description: - "Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise.", - }), - sessionAffinityFormat: optionalSessionAffinityFormat({ - description: - "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", - }), - supportsLongCacheRetention: optionalCompatBoolean({ - description: - 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', - }), - vllmPriority: Type.Optional( - Type.Number({ - description: - "vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog.", - }), - ), - }, - { + ), + supportsDeveloperRole: optionalCompatBoolean({ description: - "Compatibility settings for OpenAI-compatible completions APIs. Use this to override URL-based auto-detection for custom providers.", - additionalProperties: true, - }, -); - -export const OpenAIResponsesCompatSchema = Type.Object( - { - supportsDeveloperRole: optionalCompatBoolean({ - description: "Whether the provider supports the developer role instead of system.", + "Whether the provider supports the developer role instead of system. Default: auto-detected from URL.", + }), + supportsReasoningEffort: opt( + bool({ + description: "Whether the provider supports reasoning_effort. Default: auto-detected from URL.", + }), + ), + supportsUsageInStreaming: opt( + bool({ + description: + "Whether the provider supports stream_options.include_usage for token usage in streaming responses.", default: true, }), - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsFinishReason: opt( + bool({ description: - "Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", - default: false, - }), - sessionAffinityFormat: optionalSessionAffinityFormat({ - description: - "Session-affinity header format. openai sends session_id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", - }), - supportsLongCacheRetention: optionalCompatBoolean({ - description: - 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', + "Whether streamed responses include finish_reason. When false, pi infers stop or toolUse when the stream ends.", default: true, }), - supportsStrictMode: optionalCompatBoolean({ - description: - "Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly.", + ), + maxTokensField: opt( + union([stringLiteral("max_completion_tokens"), stringLiteral("max_tokens")], { + description: "Which field to use for max tokens. Default: auto-detected from URL.", }), - supportsOpenAIGrammarTools: optionalCompatBoolean({ + ), + requiresToolResultName: opt( + bool({ description: "Whether tool results require the name field. Default: auto-detected from URL." }), + ), + requiresAssistantAfterToolResult: opt( + bool({ description: - "Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + "Whether a user message after tool results requires an assistant message in between. Default: auto-detected from URL.", + }), + ), + requiresThinkingAsText: opt( + bool({ + description: + "Whether thinking blocks must be converted to text blocks with delimiters. Default: auto-detected from URL.", + }), + ), + requiresReasoningContentOnAssistantMessages: opt( + bool({ + description: + "Whether all replayed assistant messages must include an empty reasoning_content field when reasoning is enabled. Default: auto-detected from URL.", + }), + ), + thinkingFormat: opt( + union( + [ + stringLiteral("openai"), + stringLiteral("openrouter"), + stringLiteral("deepseek"), + stringLiteral("together"), + stringLiteral("baseten"), + stringLiteral("zai"), + stringLiteral("qwen"), + stringLiteral("chat-template"), + stringLiteral("qwen-chat-template"), + stringLiteral("string-thinking"), + stringLiteral("ant-ling"), + ], + { + description: + 'Format for reasoning or thinking parameters. When omitted, Pi auto-detects the format from the provider URL. "openai" uses reasoning_effort, "openrouter" uses reasoning.effort, "deepseek" uses thinking.type plus reasoning_effort when supported, "together" uses reasoning.enabled plus reasoning_effort when supported, "baseten" uses configurable chat_template_args plus reasoning_effort when supported, "zai" uses thinking.type, "qwen" uses top-level enable_thinking, "qwen-chat-template" uses chat_template_kwargs.enable_thinking and preserve_thinking, "chat-template" uses configurable chat_template_kwargs, "string-thinking" uses top-level thinking, and "ant-ling" uses reasoning.effort only when the mapped effort is non-null.', + }, + ), + ), + chatTemplateKwargs: opt( + stringRecord(ChatTemplateKwargValueSchema, { + description: + 'Kwargs sent as chat_template_kwargs when thinkingFormat is "chat-template". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', + }), + ), + chatTemplateArgs: opt( + stringRecord(ChatTemplateKwargValueSchema, { + description: + 'Arguments sent as chat_template_args when thinkingFormat is "baseten". Use $var with "thinking.enabled", "thinking.effort", or "thinking.budget" for pi-controlled values.', + }), + ), + openRouterRouting: opt(OpenRouterRoutingSchema), + vercelGatewayRouting: opt(VercelGatewayRoutingSchema), + zaiToolStream: opt( + bool({ + description: "Whether z.ai supports top-level tool_stream for streaming tool call deltas.", default: false, }), - supportsAdditionalTools: Type.Optional( - Type.Boolean({ - description: "Whether the model supports message-anchored additional_tools input items.", - default: false, - }), - ), - supportsToolSearch: Type.Optional( - Type.Boolean({ - description: "Whether the model supports client-executed tool search for transcript-anchored additions.", - default: false, - }), - ), - supportsExplicitPromptCacheMode: Type.Optional( - Type.Boolean({ - description: "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", - default: false, - }), - ), - supportsMidConvoEffort: Type.Optional( - Type.Boolean({ - description: - "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", - default: false, - }), - ), - supportsMaxOutputTokens: Type.Optional( - Type.Boolean({ - description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", - default: true, - }), - ), - }, - { description: "Compatibility settings for OpenAI Responses APIs.", additionalProperties: true }, -); + ), + thinkingTokenBudgetField: opt( + thinkingTokenBudgetField({ + description: + 'Top-level request field used to cap reasoning tokens from thinkingBudgets. Reasoning and the answer share max_tokens on these endpoints. "thinking_token_budget" is vLLM, "thinking_budget" is Qwen, DashScope, or SGLang, and "thinking_budget_tokens" is llama.cpp. Off by default and not set on the generated catalog.', + }), + ), + supportsThinkingTokenBudget: opt( + bool({ + description: + 'Alias for thinkingTokenBudgetField: "thinking_token_budget" (vLLM). Prefer thinkingTokenBudgetField.', + default: false, + }), + ), + supportsOpenAIGrammarTools: optionalCompatBoolean({ + description: + "Whether the provider supports OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + default: false, + }), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system or developer messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", + default: false, + }), + supportsMidConvoToolAdditions: opt( + bool({ + description: + "Whether system messages can introduce additional tools mid-conversation. Requires supportsMidConvoSystemMessages. The generated catalog enables this for capable models.", + default: false, + }), + ), + supportsStrictMode: optionalCompatBoolean({ + description: + "Whether the provider supports the strict field in tool definitions. Generated capable models enable it explicitly.", + default: false, + }), + cacheControlFormat: opt( + stringLiteral("anthropic", { + description: + "Cache control convention for prompt caching. Anthropic applies cache_control markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content.", + }), + ), + sendSessionAffinityHeaders: optionalCompatBoolean({ + description: + "Whether to send session-affinity data from options.sessionId. Default: true for OpenRouter endpoints, false otherwise.", + }), + sessionAffinityFormat: optionalSessionAffinityFormat({ + description: + "Session-affinity header format. openai sends session_id, x-client-request-id, and x-session-affinity; openai-nosession sends x-client-request-id and x-session-affinity; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", + }), + supportsLongCacheRetention: optionalCompatBoolean({ + description: + 'Whether the provider supports long prompt cache retention (prompt_cache_retention: "24h" or Anthropic-style cache_control.ttl: "1h", depending on format). Default: auto-detected from provider and URL.', + }), + vllmPriority: opt( + num({ + description: + "vLLM scheduler priority sent as the top-level priority request field. Lower values are handled earlier and the server default is 0. Only meaningful with --scheduling-policy priority. Off by default and not set on the generated catalog.", + }), + ), +}; -export const AnthropicMessagesCompatSchema = Type.Object( - { - supportsEagerToolInputStreaming: Type.Optional( - Type.Boolean({ - description: - "Whether the provider accepts per-tool eager_input_streaming. When false, the Anthropic provider omits tools[].eager_input_streaming and sends the legacy fine-grained-tool-streaming-2025-05-14 beta header for tool-enabled requests.", - default: true, - }), - ), - supportsLongCacheRetention: optionalCompatBoolean({ - description: "Whether the provider supports Anthropic long cache retention through cache_control.ttl.", +export const OpenAICompletionsCompatSchema = openObject(OpenAICompletionsCompatProperties, { + description: + "Compatibility settings for OpenAI-compatible completions APIs. Use this to override URL-based auto-detection for custom providers.", + additionalProperties: true, +}); + +const OpenAIResponsesCompatProperties = { + supportsDeveloperRole: optionalCompatBoolean({ + description: "Whether the provider supports the developer role instead of system.", + default: true, + }), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts developer or system messages after the conversation has started. When false, later system messages are folded into the leading system message. The generated catalog enables this for verified models.", + default: false, + }), + sessionAffinityFormat: optionalSessionAffinityFormat({ + description: + "Session-affinity header format. openai sends session_id and x-client-request-id; openai-nosession sends x-client-request-id; openrouter sends x-session-id. Does not affect prompt_cache_key. Default: auto-detected.", + }), + supportsLongCacheRetention: optionalCompatBoolean({ + description: + 'Whether the provider supports long prompt cache retention. This uses prompt_cache_options.ttl: "30m" on GPT-5.6+ and prompt_cache_retention: "24h" on earlier models.', + default: true, + }), + supportsStrictMode: optionalCompatBoolean({ + description: + "Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly.", + }), + supportsOpenAIGrammarTools: optionalCompatBoolean({ + description: + "Whether to emit OpenAI custom tools with Lark or regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. The generated catalog enables this for capable models.", + default: false, + }), + supportsAdditionalTools: opt( + bool({ + description: "Whether the model supports message-anchored additional_tools input items.", + default: false, + }), + ), + supportsToolSearch: opt( + bool({ + description: "Whether the model supports client-executed tool search for transcript-anchored additions.", + default: false, + }), + ), + supportsExplicitPromptCacheMode: opt( + bool({ + description: "Whether the model accepts prompt_cache_options. Older OpenAI models reject the parameter.", + default: false, + }), + ), + supportsMidConvoEffort: opt( + bool({ + description: + "Whether thinking level changes are sent as configuration_update input items instead of changing reasoning.effort, so a change keeps the prompt cache. Only used by the openai-responses API, and ignored with Sign in with ChatGPT on the openai provider.", + default: false, + }), + ), + supportsMaxOutputTokens: opt( + bool({ + description: "Whether the provider accepts max_output_tokens. Some Codex-protocol gateways reject it.", default: true, }), - sendSessionAffinityHeaders: optionalCompatBoolean({ + ), +}; + +export const OpenAIResponsesCompatSchema = openObject(OpenAIResponsesCompatProperties, { + description: "Compatibility settings for OpenAI Responses APIs.", + additionalProperties: true, +}); + +const AnthropicMessagesCompatProperties = { + supportsEagerToolInputStreaming: opt( + bool({ description: - "Whether to send x-session-affinity from options.sessionId when caching is enabled. Required for providers like Fireworks that use session affinity for prompt cache routing; requests to the same replica maximize cache hits. Default: true for OpenRouter endpoints, false otherwise.", + "Whether the provider accepts per-tool eager_input_streaming. When false, the Anthropic provider omits tools[].eager_input_streaming and sends the legacy fine-grained-tool-streaming-2025-05-14 beta header for tool-enabled requests.", + default: true, }), - sessionAffinityFormat: Type.Optional( - Type.Literal("openrouter", { - description: - "Session-affinity format. openrouter sends x-session-id; when unset, sends x-session-affinity.", - }), - ), - supportsCacheControlOnTools: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports Anthropic-style cache_control markers on tool definitions. When false, cache_control is omitted from tool parameters. Some Anthropic-compatible providers, such as Fireworks, do not support this field on tools and may reject or ignore it.", - default: true, - }), - ), - supportsTemperature: Type.Optional( - Type.Boolean({ - description: - "Whether the model accepts the Anthropic temperature request field. Claude Opus 4.7+ rejects non-default values.", - default: true, - }), - ), - forceAdaptiveThinking: Type.Optional( - Type.Boolean({ - description: - "Whether to force adaptive thinking (thinking.type: adaptive plus output_config.effort) regardless of model ID. Built-in models that require adaptive thinking set this in generated metadata. Custom Anthropic-compatible providers can set this to true for any model whose upstream requires the adaptive format. Set false to opt out on overridden built-in models.", - default: false, - }), - ), - allowEmptySignature: Type.Optional( - Type.Boolean({ - description: "Whether to replay empty thinking signatures instead of converting thinking to text.", - default: false, - }), - ), - supportsStrictTools: Type.Optional( - Type.Boolean({ - description: - "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", - default: false, - }), - ), - supportsMidConvoEffort: Type.Optional( - Type.Boolean({ - description: - "Whether the exact model transport supports effort-only system messages and thinking binding controls.", - default: false, - }), - ), - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsLongCacheRetention: optionalCompatBoolean({ + description: "Whether the provider supports Anthropic long cache retention through cache_control.ttl.", + default: true, + }), + sendSessionAffinityHeaders: optionalCompatBoolean({ + description: + "Whether to send x-session-affinity from options.sessionId when caching is enabled. Required for providers like Fireworks that use session affinity for prompt cache routing; requests to the same replica maximize cache hits. Default: true for OpenRouter endpoints, false otherwise.", + }), + sessionAffinityFormat: opt( + stringLiteral("openrouter", { + description: "Session-affinity format. openrouter sends x-session-id; when unset, sends x-session-affinity.", + }), + ), + supportsCacheControlOnTools: opt( + bool({ description: - "Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt.", + "Whether the provider supports Anthropic-style cache_control markers on tool definitions. When false, cache_control is omitted from tool parameters. Some Anthropic-compatible providers, such as Fireworks, do not support this field on tools and may reject or ignore it.", + default: true, + }), + ), + supportsTemperature: opt( + bool({ + description: + "Whether the model accepts the Anthropic temperature request field. Claude Opus 4.7+ rejects non-default values.", + default: true, + }), + ), + forceAdaptiveThinking: opt( + bool({ + description: + "Whether to force adaptive thinking (thinking.type: adaptive plus output_config.effort) regardless of model ID. Built-in models that require adaptive thinking set this in generated metadata. Custom Anthropic-compatible providers can set this to true for any model whose upstream requires the adaptive format. Set false to opt out on overridden built-in models.", default: false, }), - supportsMidConvoToolChanges: Type.Optional( - Type.Boolean({ - description: - "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.", - default: false, - }), - ), - supportsMessageThreads: optionalCompatBoolean({ - description: - "Whether the endpoint supports Anthropic message threads, which let a request with a session id send only the new messages and continue from the previous reply. Default: true for provider anthropic with a baseUrl on api.anthropic.com, false otherwise. Set true for gateways that forward the thread request field.", - }), - allowedFallbackModels: Type.Optional( - Type.Array(AnthropicAllowedFallbackModelSchema, { - maxItems: 3, - description: - "Models Anthropic accepts for server-side refusal fallback, with local pricing metadata for returned fallback responses. When absent or empty, callers must omit fallbacks; Anthropic rejects the field for models with no permitted fallback targets.", - }), - ), - }, - { description: "Compatibility settings for Anthropic Messages-compatible APIs.", additionalProperties: true }, -); - -export const BedrockCompatSchema = Type.Object( - { - supportsStrictMode: optionalCompatBoolean({ - description: "Whether the model supports Bedrock strict tool schemas.", + ), + allowEmptySignature: opt( + bool({ + description: "Whether to replay empty thinking signatures instead of converting thinking to text.", default: false, }), - }, - { description: "Compatibility settings for Amazon Bedrock models.", additionalProperties: true }, -); - -export const MistralConversationsCompatSchema = Type.Object( - { - supportsMidConvoSystemMessages: optionalCompatBoolean({ + ), + supportsStrictTools: opt( + bool({ description: - "Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message.", + "Whether the provider supports Anthropic strict tool schemas. Generated Anthropic models enable it explicitly.", default: false, }), - }, - { description: "Compatibility settings for the Mistral chat API.", additionalProperties: true }, -); - -export const SystemOneCompatSchema = Type.Object( - { - imageInput: Type.Optional( - Type.Union([Type.Literal("assets"), Type.Literal("images"), Type.Literal("state-image-urls")], { - description: - 'How the service takes classifier images. "assets" sends TypeSafe\'s assets map (PNG and JPEG) and references the images from the state, as TypeSafe and Jev on Workers AI expect. "images" sends Clef\'s images list of data URLs (PNG, JPEG, and WebP, at most four), as Clef on Workers AI expects. "state-image-urls" places image_url parts before the state, as OpenRouter expects; OpenRouter drops assets without an error. Auto-detected from the provider, baseUrl, and model ID. Classifier models read it from the provider-level compat; models and modelOverrides entries apply to chat models only.', - }), - ), - sendImageOptInHeader: optionalCompatBoolean({ + ), + supportsMidConvoEffort: opt( + bool({ description: - "Whether requests with images send the TypeSafe-Danger-Zone-Images: true header, which TypeSafe's own API requires for its experimental image input. Defaults to true for the typesafe-system-one API unless the model is detected as OpenRouter.", + "Whether the exact model transport supports effort-only system messages and thinking binding controls.", + default: false, }), - }, - { description: "Compatibility settings for System One classifier APIs.", additionalProperties: true }, -); + ), + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system-role messages inside the conversation. When false, later system messages are folded into the top-level system prompt.", + default: false, + }), + supportsMidConvoToolChanges: opt( + bool({ + description: + "Whether the exact model accepts mid-conversation tool_addition and tool_removal blocks. Requires supportsMidConvoSystemMessages.", + default: false, + }), + ), + supportsMessageThreads: optionalCompatBoolean({ + description: + "Whether the endpoint supports Anthropic message threads, which let a request with a session id send only the new messages and continue from the previous reply. Default: true for provider anthropic with a baseUrl on api.anthropic.com, false otherwise. Set true for gateways that forward the thread request field.", + }), + allowedFallbackModels: opt( + arr(AnthropicAllowedFallbackModelSchema, { + maxItems: 3, + description: + "Models Anthropic accepts for server-side refusal fallback, with local pricing metadata for returned fallback responses. When absent or empty, callers must omit fallbacks; Anthropic rejects the field for models with no permitted fallback targets.", + }), + ), +}; + +export const AnthropicMessagesCompatSchema = openObject(AnthropicMessagesCompatProperties, { + description: "Compatibility settings for Anthropic Messages-compatible APIs.", + additionalProperties: true, +}); + +const BedrockCompatProperties = { + supportsStrictMode: optionalCompatBoolean({ + description: "Whether the model supports Bedrock strict tool schemas.", + default: false, + }), +}; + +export const BedrockCompatSchema = openObject(BedrockCompatProperties, { + description: "Compatibility settings for Amazon Bedrock models.", + additionalProperties: true, +}); + +const MistralConversationsCompatProperties = { + supportsMidConvoSystemMessages: optionalCompatBoolean({ + description: + "Whether the exact model accepts system messages after the conversation has started. When false, later system messages are folded into the leading system message.", + default: false, + }), +}; + +export const MistralConversationsCompatSchema = openObject(MistralConversationsCompatProperties, { + description: "Compatibility settings for the Mistral chat API.", + additionalProperties: true, +}); + +const SystemOneCompatProperties = { + imageInput: opt( + union([stringLiteral("assets"), stringLiteral("images"), stringLiteral("state-image-urls")], { + description: + 'How the service takes classifier images. "assets" sends TypeSafe\'s assets map (PNG and JPEG) and references the images from the state, as TypeSafe and Jev on Workers AI expect. "images" sends Clef\'s images list of data URLs (PNG, JPEG, and WebP, at most four), as Clef on Workers AI expects. "state-image-urls" places image_url parts before the state, as OpenRouter expects; OpenRouter drops assets without an error. Auto-detected from the provider, baseUrl, and model ID. Classifier models read it from the provider-level compat; models and modelOverrides entries apply to chat models only.', + }), + ), + sendImageOptInHeader: optionalCompatBoolean({ + description: + "Whether requests with images send the TypeSafe-Danger-Zone-Images: true header, which TypeSafe's own API requires for its experimental image input. Defaults to true for the typesafe-system-one API unless the model is detected as OpenRouter.", + }), +}; + +export const SystemOneCompatSchema = openObject(SystemOneCompatProperties, { + description: "Compatibility settings for System One classifier APIs.", + additionalProperties: true, +}); const ProviderCompatPropertyOverrides = { supportsDeveloperRole: optionalCompatBoolean({ @@ -573,7 +585,7 @@ const ProviderCompatPropertyOverrides = { sendSessionAffinityHeaders: optionalCompatBoolean({ description: "Whether to send session-affinity data from options.sessionId. Defaults are API-specific.", }), -} satisfies TProperties; +} satisfies Properties; type UnionToIntersection = (Union extends unknown ? (value: Union) => void : never) extends ( value: infer Intersection, @@ -581,7 +593,7 @@ type UnionToIntersection = (Union extends unknown ? (value: Union) => voi ? Intersection : never; -type MergedCompatProperties = Omit< +type MergedCompatProperties = Omit< UnionToIntersection, keyof Overrides > & @@ -592,11 +604,11 @@ type MergedCompatProperties(propertyGroups: PropertyGroups, overrides: Overrides): MergedCompatProperties { - const merged: TProperties = {}; +function mergeCompatProperties( + propertyGroups: PropertyGroups, + overrides: Overrides, +): MergedCompatProperties { + const merged: Record = {}; const duplicateNames = new Set(); for (const properties of propertyGroups) { @@ -634,16 +646,25 @@ export const CompatSchemasByApi = { } as const; const ProviderCompatProperties = mergeCompatProperties( - [...new Set(Object.values(CompatSchemasByApi))].map((schema) => schema.properties), + [ + OpenAICompletionsCompatProperties, + OpenAIResponsesCompatProperties, + AnthropicMessagesCompatProperties, + BedrockCompatProperties, + MistralConversationsCompatProperties, + SystemOneCompatProperties, + ], ProviderCompatPropertyOverrides, ); -export const ProviderCompatSchema = Type.Object(ProviderCompatProperties, { +export const ProviderCompatSchema = openObject(ProviderCompatProperties, { description: "Provider and model compatibility overrides.", additionalProperties: true, }); export type ChatTemplateKwargValue = Static; +/** Provider and model compatibility overrides in models.json: every API's compat fields. */ +export type ProviderCompat = Static; /** Top-level request field used to cap reasoning tokens on OpenAI-compatible servers. */ export type ThinkingTokenBudgetField = Static; export type SessionAffinityFormat = Static; diff --git a/packages/ai/src/providers/github-copilot.ts b/packages/ai/src/providers/github-copilot.ts index eab784d88..282bc1da1 100644 --- a/packages/ai/src/providers/github-copilot.ts +++ b/packages/ai/src/providers/github-copilot.ts @@ -1,13 +1,28 @@ import { anthropicMessagesApi } from "../api/anthropic-messages.lazy.ts"; import { openAICompletionsApi } from "../api/openai-completions.lazy.ts"; import { openAIResponsesApi } from "../api/openai-responses.lazy.ts"; +import { typesafeSystemOneApi } from "../api/typesafe-system-one.lazy.ts"; import { envApiKeyAuth, lazyOAuth } from "../auth/helpers.ts"; import { loadGitHubCopilotOAuth } from "../auth/oauth/load.ts"; import { createProvider, type Provider } from "../models.ts"; -import { GITHUB_COPILOT_MODELS } from "./github-copilot.models.ts"; +import type { ProviderClassifier } from "../types.ts"; +import { GITHUB_COPILOT_CLASSIFIER_MODELS, GITHUB_COPILOT_MODELS } from "./github-copilot.models.ts"; -export function githubCopilotProvider(): Provider<"anthropic-messages" | "openai-completions" | "openai-responses"> { - return createProvider({ +type GitHubCopilotApi = "anthropic-messages" | "openai-completions" | "openai-responses"; + +/** + * Copilot serves System One at `/v1/systemone`, while its chat APIs have no version prefix. + * The login replaces the catalog base URL with the account's endpoint, so the prefix is added per request. + */ +function copilotSystemOne(classifier: ProviderClassifier): ProviderClassifier { + return { + classify: (model, context, options) => + classifier.classify({ ...model, baseUrl: `${model.baseUrl.replace(/\/+$/u, "")}/v1` }, context, options), + }; +} + +export function githubCopilotProvider(): Provider { + return createProvider({ id: "github-copilot", name: "GitHub Copilot", baseUrl: "https://api.individual.githubcopilot.com", @@ -15,7 +30,7 @@ export function githubCopilotProvider(): Provider<"anthropic-messages" | "openai apiKey: envApiKeyAuth("GitHub Copilot token", ["COPILOT_GITHUB_TOKEN"]), oauth: lazyOAuth({ name: "GitHub Copilot", isSubscription: true, load: loadGitHubCopilotOAuth }), }, - models: Object.values(GITHUB_COPILOT_MODELS), + models: [...Object.values(GITHUB_COPILOT_MODELS), ...Object.values(GITHUB_COPILOT_CLASSIFIER_MODELS)], filterModels: (models, credential) => { if (credential?.type !== "oauth") return models; const availableModelIds = credential.availableModelIds; @@ -30,5 +45,6 @@ export function githubCopilotProvider(): Provider<"anthropic-messages" | "openai "openai-completions": openAICompletionsApi(), "openai-responses": openAIResponsesApi(), }, + classifiers: { "typesafe-system-one": copilotSystemOne(typesafeSystemOneApi()) }, }); } diff --git a/packages/ai/src/providers/model-schema.ts b/packages/ai/src/providers/model-schema.ts index 453ceabc7..2eb0a5a52 100644 --- a/packages/ai/src/providers/model-schema.ts +++ b/packages/ai/src/providers/model-schema.ts @@ -1,4 +1,17 @@ -import { type Static, type TSchemaOptions, Type } from "typebox"; +import { + type Annotations, + arr, + int, + nul, + num, + oneOf, + opt, + type Static, + type StaticObject, + str, + union, +} from "@earendil-works/pi-schema"; +import { openObject as object } from "./schema-builders.ts"; export const THINKING_LEVELS = ["minimal", "low", "medium", "high", "xhigh", "max"] as const; export const MODEL_THINKING_LEVELS = ["off", ...THINKING_LEVELS] as const; @@ -6,19 +19,31 @@ export const MODEL_INPUT_MODALITIES = ["text", "image"] as const; export const CACHE_RETENTIONS = ["none", "short", "long"] as const; export const MODEL_PROMPT_CACHE_RETENTIONS = ["short", "long"] as const; -export const ThinkingLevelSchema = Type.Enum(THINKING_LEVELS); -export const ModelThinkingLevelSchema = Type.Enum(MODEL_THINKING_LEVELS); -export const ModelInputModalitySchema = Type.Enum(MODEL_INPUT_MODALITIES); -export const CacheRetentionSchema = Type.Enum(CACHE_RETENTIONS); +export const ThinkingLevelSchema = oneOf(THINKING_LEVELS); +export const ModelThinkingLevelSchema = oneOf(MODEL_THINKING_LEVELS); +export const ModelInputModalitySchema = oneOf(MODEL_INPUT_MODALITIES); +export const CacheRetentionSchema = oneOf(CACHE_RETENTIONS); -const ThinkingLevelMapValueSchema = Type.Union([Type.String(), Type.Null()]); -export const ThinkingLevelMapSchema = Type.Partial(Type.Record(ModelThinkingLevelSchema, ThinkingLevelMapValueSchema), { - description: - "Provider value sent for each thinking level. null marks a level unsupported. Levels the model does not map use the provider's default value through high; xhigh and max are unsupported.", -}); +const ThinkingLevelMapValueSchema = union([str(), nul()]); +export const ThinkingLevelMapSchema = object( + { + off: opt(ThinkingLevelMapValueSchema), + minimal: opt(ThinkingLevelMapValueSchema), + low: opt(ThinkingLevelMapValueSchema), + medium: opt(ThinkingLevelMapValueSchema), + high: opt(ThinkingLevelMapValueSchema), + xhigh: opt(ThinkingLevelMapValueSchema), + max: opt(ThinkingLevelMapValueSchema), + }, + { + description: + "Provider value sent for each thinking level. null marks a level unsupported. Levels the model does not map use the provider's default value through high; xhigh and max are unsupported.", + }, +); -export const ModelPromptCacheSchema = Type.Partial( - Type.Record(Type.Enum(MODEL_PROMPT_CACHE_RETENTIONS), Type.Number({ exclusiveMinimum: 0 })), +const PromptCacheLifetimeSchema = num({ exclusiveMinimum: 0 }); +export const ModelPromptCacheSchema = object( + { short: opt(PromptCacheLifetimeSchema), long: opt(PromptCacheLifetimeSchema) }, { description: "Best-effort prompt cache lifetime in seconds for the short and long retention tiers. Pi warms only tiers with a known lifetime.", @@ -26,63 +51,54 @@ export const ModelPromptCacheSchema = Type.Partial( ); const ModelCostRatesProperties = { - input: Type.Number({ description: "Input cost in USD per million tokens." }), - output: Type.Number({ description: "Output cost in USD per million tokens." }), - cacheRead: Type.Number({ description: "Cache-read cost in USD per million tokens." }), - cacheWrite: Type.Number({ description: "Cache-write cost in USD per million tokens." }), + input: num({ description: "Input cost in USD per million tokens." }), + output: num({ description: "Output cost in USD per million tokens." }), + cacheRead: num({ description: "Cache-read cost in USD per million tokens." }), + cacheWrite: num({ description: "Cache-write cost in USD per million tokens." }), }; -export const ModelCostRatesSchema = Type.Object(ModelCostRatesProperties); -export const ModelCostTierSchema = Type.Object({ - inputTokensAbove: Type.Number({ +export const ModelCostRatesSchema = object(ModelCostRatesProperties); +export const ModelCostTierSchema = object({ + inputTokensAbove: num({ description: "Use this tier when total request input exceeds this token count.", }), ...ModelCostRatesProperties, }); -export const ModelCostSchema = Type.Object({ +export const ModelCostSchema = object({ ...ModelCostRatesProperties, - tiers: Type.Optional( - Type.Array(ModelCostTierSchema, { + tiers: opt( + arr(ModelCostTierSchema, { description: "Request-wide pricing tiers. The highest matching input threshold applies to the full request.", }), ), }); -function modelImageResizeOptions(options?: TSchemaOptions) { - return Type.Object( - { - maxWidth: Type.Optional(Type.Integer({ minimum: 1 })), - maxHeight: Type.Optional(Type.Integer({ minimum: 1 })), - maxBytes: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum base64-encoded payload size in bytes." }), - ), - jpegQuality: Type.Optional(Type.Integer({ minimum: 1, maximum: 100 })), - }, - options, - ); +const ModelImageResizeProperties = { + maxWidth: opt(int({ minimum: 1 })), + maxHeight: opt(int({ minimum: 1 })), + maxBytes: opt(int({ minimum: 1, description: "Maximum base64-encoded payload size in bytes." })), + jpegQuality: opt(int({ minimum: 1, maximum: 100 })), +}; + +function modelImageResizeOptions(options?: Annotations>) { + return object(ModelImageResizeProperties, options); } export const ModelImageResizeOptionsSchema = modelImageResizeOptions(); -export const ModelImageInputLimitsSchema = Type.Object({ - resize: Type.Optional( +export const ModelImageInputLimitsSchema = object({ + resize: opt( modelImageResizeOptions({ description: "Cache-safe resize profile applied before a new image enters conversation history.", }), ), - maxPerMessage: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum images accepted in one provider message." }), - ), - maxPerRequest: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum images accepted across one provider request." }), - ), + maxPerMessage: opt(int({ minimum: 1, description: "Maximum images accepted in one provider message." })), + maxPerRequest: opt(int({ minimum: 1, description: "Maximum images accepted across one provider request." })), }); -export const ModelInputLimitsSchema = Type.Object({ - maxRequestBytes: Type.Optional( - Type.Integer({ minimum: 1, description: "Maximum serialized provider request size in bytes." }), - ), - images: Type.Optional(ModelImageInputLimitsSchema), +export const ModelInputLimitsSchema = object({ + maxRequestBytes: opt(int({ minimum: 1, description: "Maximum serialized provider request size in bytes." })), + images: opt(ModelImageInputLimitsSchema), }); export type ThinkingLevel = Static; diff --git a/packages/ai/src/providers/schema-builders.ts b/packages/ai/src/providers/schema-builders.ts new file mode 100644 index 000000000..2df034ca4 --- /dev/null +++ b/packages/ai/src/providers/schema-builders.ts @@ -0,0 +1,34 @@ +// Builders for pi-ai's own configuration schemas (model metadata, compat settings): pi-schema's builders, shaped like the +// TypeBox schemas they replace, so models.json and the catalog validate exactly as before. + +import { + type Annotations, + obj, + type Properties, + type Schema, + type Static, + type StaticObject, +} from "@earendil-works/pi-schema"; + +/** An object that allows other properties; `additionalProperties: true` says so explicitly. */ +export function openObject( + properties: P, + options: Annotations> & { readonly additionalProperties?: true } = {}, +): Schema> { + const { additionalProperties, ...annotations } = options; + const schema = obj(properties, { ...annotations, open: true }); + return additionalProperties ? { ...schema, additionalProperties } : schema; +} + +/** One string: `{ type: "string", const: value }`. */ +export function stringLiteral(value: V, options: Annotations = {}): Schema { + return { type: "string", const: value, ...options }; +} + +/** An object with any string keys and values of `value`: `{ type: "object", patternProperties: { "^.*$": value } }`. */ +export function stringRecord( + value: S, + options: Annotations>> = {}, +): Schema>> { + return { type: "object", patternProperties: { "^.*$": value }, ...options }; +} diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index babda62fc..c1ec24dcb 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -749,8 +749,6 @@ export interface ClassifierResult { timestamp: number; // Unix timestamp in milliseconds } -import type { TSchema } from "typebox"; - /** OpenAI grammar variants for constrained sampling. */ export type GrammarFormat = "openai_lark" | "openai_regex"; @@ -773,7 +771,11 @@ export type ConstrainedSamplingConfig = variants: GrammarVariants; }; -export interface Tool { +/** + * A tool the model may call. `parameters` is the JSON Schema of its arguments: pi-schema's builders (which also type + * the arguments, see `Static` in `@earendil-works/pi-schema`), TypeBox schemas, or any JSON Schema object. + */ +export interface Tool { name: string; description: string; parameters: TParameters; diff --git a/packages/ai/src/utils/retry.ts b/packages/ai/src/utils/retry.ts index a1e984032..eabf6c7de 100644 --- a/packages/ai/src/utils/retry.ts +++ b/packages/ai/src/utils/retry.ts @@ -28,6 +28,10 @@ const NON_RETRYABLE_PROVIDER_LIMIT_ERROR_PATTERN = buildProviderErrorPattern([ // Sign in with ChatGPT on a ChatGPT Business plan: a usage limit set by the // workspace admin. Only the admin can raise it. "subscription_sharing_workplace_individual_usage_limit_exceeded", + + // Zhipu/Z.ai (open.bigmodel.cn) error code 1308: the plan's usage limit is + // reached, sent as HTTP 429. It resets after hours, not seconds (#11537). + '"code"\\s*:\\s*"?1308(?![0-9])', ]); const RETRYABLE_PROVIDER_ERROR_PATTERN = buildProviderErrorPattern([ diff --git a/packages/ai/src/utils/typebox-helpers.ts b/packages/ai/src/utils/string-enum.ts similarity index 68% rename from packages/ai/src/utils/typebox-helpers.ts rename to packages/ai/src/utils/string-enum.ts index 85a53555c..b660178f8 100644 --- a/packages/ai/src/utils/typebox-helpers.ts +++ b/packages/ai/src/utils/string-enum.ts @@ -1,24 +1,24 @@ -import { type TUnsafe, Type } from "typebox"; +import type { Schema } from "@earendil-works/pi-schema"; /** * Creates a string enum schema compatible with Google's API and other providers - * that don't support anyOf/const patterns. + * that don't support anyOf/const patterns: `{ type: "string", enum: [...] }`. * * @example * const OperationSchema = StringEnum(["add", "subtract", "multiply", "divide"], { * description: "The operation to perform" * }); * - * type Operation = Static; // "add" | "subtract" | "multiply" | "divide" + * type Operation = Static; // "add" | "subtract" | "multiply" | "divide" (Static from @earendil-works/pi-schema) */ export function StringEnum( values: T, options?: { description?: string; default?: T[number] }, -): TUnsafe { - return Type.Unsafe({ +): Schema { + return { type: "string", - enum: values as any, + enum: values, ...(options?.description && { description: options.description }), ...(options?.default && { default: options.default }), - }); + }; } diff --git a/packages/ai/src/utils/transcript.ts b/packages/ai/src/utils/transcript.ts index 6f8a2691f..e9bec79f8 100644 --- a/packages/ai/src/utils/transcript.ts +++ b/packages/ai/src/utils/transcript.ts @@ -133,7 +133,7 @@ export function toToolDeclaration(tool: Tool): Tool { * Whether two tools declare the same interface to the model. * * Both sides go through {@link toToolDeclaration} first: its JSON round-trip drops the - * typebox symbol keys and `undefined` fields that a structural comparison would see, and + * schema metadata keys (TypeBox's non-enumerable kind marks) and `undefined` fields that a structural comparison would see, and * builds both objects with the same key order, so comparing the serialized declarations * is exact. This avoids a deep-equal dependency in a browser-safe package. */ diff --git a/packages/ai/src/utils/validation.ts b/packages/ai/src/utils/validation.ts index c9e818834..586765bf0 100644 --- a/packages/ai/src/utils/validation.ts +++ b/packages/ai/src/utils/validation.ts @@ -1,299 +1,34 @@ -import { Compile } from "typebox/compile"; -import type { TLocalizedValidationError } from "typebox/error"; -import { Value } from "typebox/value"; +import { type CoercingValidator, compile } from "@earendil-works/pi-schema"; +import { formats } from "@earendil-works/pi-schema/formats"; import type { Tool, ToolCall } from "../types.ts"; -const validatorCache = new WeakMap>(); -const TYPEBOX_KIND = Symbol.for("TypeBox.Kind"); +// One compiled validator per schema object: tool schemas are long-lived, so the WeakMap drops entries with their tools. +const validatorCache = new WeakMap(); -interface JsonSchemaObject { - type?: string | string[]; - properties?: Record; - required?: string[]; - items?: JsonSchemaObject | JsonSchemaObject[]; - additionalProperties?: boolean | JsonSchemaObject; - allOf?: JsonSchemaObject[]; - anyOf?: JsonSchemaObject[]; - oneOf?: JsonSchemaObject[]; -} - -function getSchemaTypes(schema: JsonSchemaObject): string[] { - if (typeof schema.type === "string") { - return [schema.type]; - } - if (Array.isArray(schema.type)) { - return schema.type.filter((type): type is string => typeof type === "string"); - } - return []; -} - -function matchesJsonType(value: unknown, type: string): boolean { - switch (type) { - case "number": - return typeof value === "number"; - case "integer": - return typeof value === "number" && Number.isInteger(value); - case "boolean": - return typeof value === "boolean"; - case "string": - return typeof value === "string"; - case "null": - return value === null; - case "array": - return Array.isArray(value); - case "object": - return typeof value === "object" && value !== null && !Array.isArray(value); - default: - return false; - } -} - -function getSubSchemaValidator(schema: JsonSchemaObject): ReturnType | undefined { - try { - return getValidator(schema as Tool["parameters"]); - } catch { - return undefined; - } -} - -function coercePrimitiveByType(value: unknown, type: string): unknown { - switch (type) { - case "number": { - if (value === null) { - return 0; - } - if (typeof value === "string" && value.trim() !== "") { - const parsed = Number(value); - if (Number.isFinite(parsed)) { - return parsed; - } - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - return value; - } - case "integer": { - if (value === null) { - return 0; - } - if (typeof value === "string" && value.trim() !== "") { - const parsed = Number(value); - if (Number.isInteger(parsed)) { - return parsed; - } - } - if (typeof value === "boolean") { - return value ? 1 : 0; - } - return value; - } - case "boolean": { - if (value === null) { - return false; - } - if (typeof value === "string") { - if (value === "true") { - return true; - } - if (value === "false") { - return false; - } - } - if (typeof value === "number") { - if (value === 1) { - return true; - } - if (value === 0) { - return false; - } - } - return value; - } - case "string": { - if (value === null) { - return ""; - } - if (typeof value === "number" || typeof value === "boolean") { - return String(value); - } - return value; - } - case "null": { - if (value === "" || value === 0 || value === false) { - return null; - } - return value; - } - default: - return value; - } -} - -function applySchemaObjectCoercion(value: Record, schema: JsonSchemaObject): void { - const properties = schema.properties; - const definedKeys = new Set(properties ? Object.keys(properties) : []); - - if (properties) { - for (const [key, propertySchema] of Object.entries(properties)) { - if (!(key in value)) { - continue; - } - value[key] = coerceWithJsonSchema(value[key], propertySchema); - } - } - - if (schema.additionalProperties && typeof schema.additionalProperties === "object") { - for (const [key, propertyValue] of Object.entries(value)) { - if (definedKeys.has(key)) { - continue; - } - value[key] = coerceWithJsonSchema(propertyValue, schema.additionalProperties); - } - } -} - -function applySchemaArrayCoercion(value: unknown[], schema: JsonSchemaObject): void { - if (Array.isArray(schema.items)) { - for (let index = 0; index < value.length; index++) { - const itemSchema = schema.items[index]; - if (!itemSchema) { - continue; - } - value[index] = coerceWithJsonSchema(value[index], itemSchema); - } - return; - } - - if (schema.items && typeof schema.items === "object") { - for (let index = 0; index < value.length; index++) { - value[index] = coerceWithJsonSchema(value[index], schema.items); - } - } -} - -function coerceWithUnionSchema(value: unknown, schemas: JsonSchemaObject[]): unknown { - for (const schema of schemas) { - const validator = getSubSchemaValidator(schema); - if (validator?.Check(value)) { - return value; - } - } - - for (const schema of schemas) { - const candidate = structuredClone(value); - const coerced = coerceWithJsonSchema(candidate, schema); - const validator = getSubSchemaValidator(schema); - if (validator?.Check(coerced)) { - return coerced; - } - } - return value; -} - -function coerceWithJsonSchema(value: unknown, schema: JsonSchemaObject): unknown { - let nextValue = value; - - if (Array.isArray(schema.allOf)) { - for (const nested of schema.allOf) { - nextValue = coerceWithJsonSchema(nextValue, nested); - } - } - - if (Array.isArray(schema.anyOf)) { - nextValue = coerceWithUnionSchema(nextValue, schema.anyOf); - } - - if (Array.isArray(schema.oneOf)) { - nextValue = coerceWithUnionSchema(nextValue, schema.oneOf); - } - - const schemaTypes = getSchemaTypes(schema); - const matchesUnionMember = - schemaTypes.length > 1 && schemaTypes.some((schemaType) => matchesJsonType(nextValue, schemaType)); - if (schemaTypes.length > 0 && !matchesUnionMember) { - for (const schemaType of schemaTypes) { - const candidate = coercePrimitiveByType(nextValue, schemaType); - if (candidate !== nextValue) { - nextValue = candidate; - break; - } - } - } - - if ( - schemaTypes.includes("object") && - typeof nextValue === "object" && - nextValue !== null && - !Array.isArray(nextValue) - ) { - applySchemaObjectCoercion(nextValue as Record, schema); - } - - if (schemaTypes.includes("array") && Array.isArray(nextValue)) { - applySchemaArrayCoercion(nextValue, schema); - } - - return nextValue; -} - -function normalizeOptionalNulls(value: unknown, schema: JsonSchemaObject): void { - if (Array.isArray(value)) { - if (Array.isArray(schema.items)) { - for (let index = 0; index < value.length; index++) { - const itemSchema = schema.items[index]; - if (itemSchema) normalizeOptionalNulls(value[index], itemSchema); - } - } else if (schema.items) { - for (const item of value) normalizeOptionalNulls(item, schema.items); - } - return; - } - if (typeof value !== "object" || value === null || !schema.properties) return; - - const object = value as Record; - const required = new Set(schema.required ?? []); - for (const [key, propertySchema] of Object.entries(schema.properties)) { - if (!(key in object)) continue; - if ( - object[key] === null && - !required.has(key) && - typeof (propertySchema as { $ref?: unknown }).$ref !== "string" && - getSubSchemaValidator(propertySchema)?.Check(null) === false - ) { - delete object[key]; - } else { - normalizeOptionalNulls(object[key], propertySchema); - } - } -} - -function getValidator(schema: Tool["parameters"]): ReturnType { - const key = schema as object; - const cached = validatorCache.get(key); +function getValidator(schema: Tool["parameters"]): CoercingValidator { + const cached = validatorCache.get(schema); if (cached) { return cached; } - const validator = Compile(schema); - validatorCache.set(key, validator); + // A $ref that names nothing (a dangling definition in an MCP server's schema) rejects only values at its place, + // instead of making every call of the tool fail. + const validator = compile(schema, { formats, coerce: true, unresolvedRefs: "reject" }); + validatorCache.set(schema, validator); return validator; } -function formatValidationPath(error: TLocalizedValidationError): string { - if (error.keyword === "required") { - const requiredProperties = (error.params as { requiredProperties?: string[] }).requiredProperties; - const requiredProperty = requiredProperties?.[0]; - if (requiredProperty) { - const basePath = error.instancePath.replace(/^\//, "").replace(/\//g, "."); - return basePath ? `${basePath}.${requiredProperty}` : requiredProperty; - } - } - const path = error.instancePath.replace(/^\//, "").replace(/\//g, "."); - return path || "root"; +/** A pi-schema error path (a JSON Pointer) as the dotted path the model sees: `/edits/0/oldText` -> `edits.0.oldText`. */ +function formatValidationPath(pointer: string): string { + if (pointer === "") return "root"; + return pointer + .slice(1) + .split("/") + .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~")) + .join("."); } /** - * Finds a tool by name and validates the tool call arguments against its TypeBox schema + * Finds a tool by name and validates the tool call arguments against its JSON Schema * @param tools Array of tool definitions * @param toolCall The tool call from the LLM * @returns The validated arguments @@ -308,8 +43,10 @@ export function validateToolCall(tools: Tool[], toolCall: ToolCall): any { } /** - * Validates tool call arguments against the tool's TypeBox schema - * @param tool The tool definition with TypeBox schema + * Validates tool call arguments against the tool's JSON Schema (any JSON Schema: pi-schema's builders, TypeBox, plain + * objects). The arguments are coerced first (pi-schema's `parse()`): models often send `"42"` for a number, a single + * value for an array, or `null` for an omitted optional property. + * @param tool The tool definition with its JSON Schema * @param toolCall The tool call from the LLM * @returns The validated (and potentially coerced) arguments * @throws Error with formatted message if validation fails @@ -318,34 +55,13 @@ export function validateToolArguments(tool: Tool, toolCall: ToolCall): any { if (toolCall.argumentsError) { throw new Error(`Invalid arguments for tool "${toolCall.name}": ${toolCall.argumentsError}`); } - const args = structuredClone(toolCall.arguments); - normalizeOptionalNulls(args, tool.parameters as JsonSchemaObject); - Value.Convert(tool.parameters, args); - - const validator = getValidator(tool.parameters); - if (!Object.getOwnPropertySymbols(tool.parameters).includes(TYPEBOX_KIND)) { - const coerced = coerceWithJsonSchema(args, tool.parameters as JsonSchemaObject); - if (coerced !== args) { - if (typeof args === "object" && args !== null && typeof coerced === "object" && coerced !== null) { - for (const key of Object.keys(args)) { - delete args[key]; - } - Object.assign(args, coerced); - } else { - return validator.Check(coerced) ? coerced : args; - } - } + // A copy, so tools that modify their arguments never modify the tool call in the transcript. + const result = getValidator(tool.parameters).parse(structuredClone(toolCall.arguments)); + if (result.ok) { + return result.value; } - if (validator.Check(args)) { - return args; - } - - const errors = - validator - .Errors(args) - .map((error) => ` - ${formatValidationPath(error)}: ${error.message}`) - .join("\n") || "Unknown validation error"; + const errors = result.errors.map((error) => ` - ${formatValidationPath(error.path)}: ${error.message}`).join("\n"); const errorMessage = `Validation failed for tool "${toolCall.name}":\n${errors}\n\nReceived arguments:\n${JSON.stringify(toolCall.arguments, null, 2)}`; diff --git a/packages/ai/src/utils/version.ts b/packages/ai/src/utils/version.ts index 18fe5a582..f1e6f3211 100644 --- a/packages/ai/src/utils/version.ts +++ b/packages/ai/src/utils/version.ts @@ -1,2 +1,2 @@ // The @earendil-works/pi-ai package version. scripts/sync-versions.js keeps it in sync with package.json. -export const PI_AI_VERSION = "1.4.0"; +export const PI_AI_VERSION = "1.5.0"; diff --git a/packages/ai/test/anthropic-strict-tool-schema.test.ts b/packages/ai/test/anthropic-strict-tool-schema.test.ts index 587cde17f..0e7ec623d 100644 --- a/packages/ai/test/anthropic-strict-tool-schema.test.ts +++ b/packages/ai/test/anthropic-strict-tool-schema.test.ts @@ -98,6 +98,31 @@ describe("Anthropic strict tool schemas", () => { }); }); + // https://github.com/earendil-works/pi/issues/9557 + it("keeps root definitions in non-strict schemas so property refs resolve", async () => { + const settings = { type: "object", properties: { verbose: { type: "boolean" } } }; + const cases = [ + { ref: "#/$defs/Settings", definitions: { $defs: { Settings: settings } } }, + { ref: "#/definitions/Settings", definitions: { definitions: { Settings: settings } } }, + ]; + for (const { ref, definitions } of cases) { + const parameters = { + type: "object", + properties: { settings: { $ref: ref } }, + required: ["settings"], + ...definitions, + } as unknown as Tool["parameters"]; + const tool = await captureFirstTool(createTool(parameters)); + expect(tool.strict).toBeUndefined(); + expect(tool.input_schema).toEqual({ + type: "object", + properties: { settings: { $ref: ref } }, + required: ["settings"], + ...definitions, + }); + } + }); + // https://github.com/earendil-works/pi/issues/9953 it("sends prefer tools non-strict when they use keywords Anthropic strict mode rejects", async () => { const unsupportedParameters: Tool["parameters"][] = [ diff --git a/packages/ai/test/clamp-max-tokens.test.ts b/packages/ai/test/clamp-max-tokens.test.ts new file mode 100644 index 000000000..ada7a22a4 --- /dev/null +++ b/packages/ai/test/clamp-max-tokens.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { clampMaxTokensToContext } from "../src/api/simple-options.ts"; +import { getModel } from "../src/compat.ts"; +import type { AssistantMessage, Model } from "../src/types.ts"; +import { normalizeContext } from "../src/utils/transcript.ts"; + +const WINDOW = 1_048_576; + +function modelWith(maxTokens: number, contextWindow = WINDOW): Model<"openai-completions"> { + const { compat: _compat, ...base } = getModel("openai", "gpt-4o-mini")!; + return { ...base, api: "openai-completions", contextWindow, maxTokens }; +} + +// The last assistant usage anchors the estimate, so the estimated input is exactly `inputTokens`. +function contextWithInput(inputTokens: number) { + const assistant: AssistantMessage = { + role: "assistant", + content: [{ type: "text", text: "ok" }], + api: "openai-completions", + provider: "openai", + model: "gpt-4o-mini", + usage: { + input: inputTokens, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: inputTokens, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: 2, + }; + return normalizeContext({ messages: [{ role: "user", content: "hi", timestamp: 1 }, assistant] }); +} + +describe("clampMaxTokensToContext", () => { + it("leaves 15% of the input unused so an OpenRouter overcount still fits (#10497)", () => { + // The failing request from #10497: OpenRouter counted 229,747 input tokens where pi estimated 220,815. + const maxTokens = clampMaxTokensToContext(modelWith(943_718), contextWithInput(220_815), 943_718); + expect(maxTokens).toBe(WINDOW - 220_815 - 33_123); + expect(229_747 + maxTokens).toBeLessThanOrEqual(WINDOW); + }); + + it("leaves at least 4096 tokens unused for small inputs", () => { + expect(clampMaxTokensToContext(modelWith(WINDOW), contextWithInput(10_000), WINDOW)).toBe(WINDOW - 10_000 - 4096); + }); + + it("keeps the model output limit when it already fits", () => { + expect(clampMaxTokensToContext(modelWith(64_000), contextWithInput(100_000), 64_000)).toBe(64_000); + }); + + it("sends up to 32768 tokens when the proportional margin leaves no room", () => { + expect(clampMaxTokensToContext(modelWith(943_718), contextWithInput(912_000), 943_718)).toBe(32_768); + }); + + it("uses the real room minus 4096 when less than 32768 tokens fit", () => { + expect(clampMaxTokensToContext(modelWith(943_718), contextWithInput(1_030_000), 943_718)).toBe( + WINDOW - 1_030_000 - 4096, + ); + }); + + it("keeps only the fixed 4096 margin for small windows", () => { + expect(clampMaxTokensToContext(modelWith(32_768, 32_768), contextWithInput(20_000), 32_768)).toBe( + 32_768 - 20_000 - 4096, + ); + }); + + it("sends 1 token when the input already exceeds the window", () => { + expect(clampMaxTokensToContext(modelWith(943_718), contextWithInput(1_050_000), 943_718)).toBe(1); + }); +}); diff --git a/packages/ai/test/classifier-models.test.ts b/packages/ai/test/classifier-models.test.ts index cae1df4a5..9f26b199c 100644 --- a/packages/ai/test/classifier-models.test.ts +++ b/packages/ai/test/classifier-models.test.ts @@ -245,6 +245,62 @@ describe("Models with classifier models", () => { expect((await withOAuth.getAvailable("openai")).some((model) => model.id === "gpt-6-luna")).toBe(true); }); + it("routes GitHub Copilot Microsoft-Decision-1 to the account's /v1/systemone endpoint", async () => { + const credentials = new InMemoryCredentialStore(); + await credentials.modify("github-copilot", async () => ({ + type: "oauth", + access: "tid=test;exp=9999999999;proxy-ep=proxy.enterprise.githubcopilot.com;", + refresh: "refresh", + expires: Date.now() + 3_600_000, + // The login records only picker models; classifiers stay listed. + availableModelIds: ["gpt-6-luna"], + })); + const models = builtinModels({ credentials }); + const decision = models.getModelOfType("classifier", "github-copilot", "microsoft-decision-1"); + if (!decision) throw new Error("missing GitHub Copilot decision model"); + expect(decision).toMatchObject({ + api: "typesafe-system-one", + baseUrl: "https://api.individual.githubcopilot.com", + input: ["text"], + contextWindow: 32768, + }); + expect(models.getModel("github-copilot", "microsoft-decision-1")).toBeUndefined(); + expect(await models.getAvailableOfType("classifier", "github-copilot")).toEqual([decision]); + + const requests: Array<{ url: string; headers: Headers }> = []; + const result = await models.classify(decision, context, { + fetch: async (input, init) => { + requests.push({ url: String(input), headers: new Headers(init?.headers) }); + return Response.json({ answers: { approved: { type: "noul", noul: 0.8 } } }); + }, + }); + + expect(requests.map((request) => request.url)).toEqual(["https://api.enterprise.githubcopilot.com/v1/systemone"]); + expect(requests[0].headers.get("authorization")).toBe( + "Bearer tid=test;exp=9999999999;proxy-ep=proxy.enterprise.githubcopilot.com;", + ); + // Copilot rejects requests without an editor version. + expect(requests[0].headers.get("editor-version")).toBeTruthy(); + expect(requests[0].headers.get("copilot-integration-id")).toBe("vscode-chat"); + expect(result.stopReason).toBe("stop"); + expect(result.answers.approved).toEqual({ type: "bool", probability: 0.8 }); + }); + + it("adds the System One version prefix to the GitHub Copilot catalog base URL", async () => { + const models = builtinModels(); + const decision = models.getModelOfType("classifier", "github-copilot", "microsoft-decision-1"); + if (!decision) throw new Error("missing GitHub Copilot decision model"); + const urls: string[] = []; + await models.classify(decision, context, { + apiKey: "secret", + fetch: async (input) => { + urls.push(String(input)); + return Response.json({ answers: { approved: { type: "noul", noul: 0.8 } } }); + }, + }); + expect(urls).toEqual(["https://api.individual.githubcopilot.com/v1/systemone"]); + }); + it("routes OpenRouter classifier models through the System One API", () => { const models = builtinModels(); for (const model of getBuiltinClassifierModels("openrouter")) { diff --git a/packages/ai/test/compat-schema.test.ts b/packages/ai/test/compat-schema.test.ts index 4ef40713e..d1a6d66ca 100644 --- a/packages/ai/test/compat-schema.test.ts +++ b/packages/ai/test/compat-schema.test.ts @@ -1,5 +1,4 @@ -import type { Static } from "typebox"; -import { Compile } from "typebox/compile"; +import type { Static } from "@earendil-works/pi-schema"; import { describe, expect, expectTypeOf, it } from "vitest"; import { AnthropicMessagesCompatSchema, @@ -11,6 +10,16 @@ import { type ProviderCompat = Static; +/** The `default` annotation of each property of an object schema. */ +function defaults(schema: { readonly [keyword: string]: unknown }): Record { + const properties = schema.properties as Record; + return Object.fromEntries( + Object.entries(properties) + .filter(([, property]) => property.default !== undefined) + .map(([name, property]) => [name, property.default]), + ); +} + describe("compatibility schemas", () => { it("preserves property types in the provider superset", () => { expectTypeOf().toEqualTypeOf(); @@ -20,21 +29,21 @@ describe("compatibility schemas", () => { }); it("preserves API-specific defaults", () => { - const completionsDefaults = Compile(OpenAICompletionsCompatSchema).Default({}); + const completionsDefaults = defaults(OpenAICompletionsCompatSchema); expect(completionsDefaults).not.toHaveProperty("thinkingFormat"); expect(completionsDefaults).not.toHaveProperty("supportsLongCacheRetention"); - expect(Compile(OpenAIResponsesCompatSchema).Default({})).toMatchObject({ supportsDeveloperRole: true }); - const anthropicDefaults = Compile(AnthropicMessagesCompatSchema).Default({}); + expect(defaults(OpenAIResponsesCompatSchema)).toMatchObject({ supportsDeveloperRole: true }); + const anthropicDefaults = defaults(AnthropicMessagesCompatSchema); expect(anthropicDefaults).not.toHaveProperty("sendSessionAffinityHeaders"); expect(anthropicDefaults).toMatchObject({ supportsLongCacheRetention: true }); - expect(Compile(BedrockCompatSchema).Default({})).toMatchObject({ supportsStrictMode: false }); + expect(defaults(BedrockCompatSchema)).toMatchObject({ supportsStrictMode: false }); }); it("does not assign API-specific defaults to the provider superset", () => { - const defaults = Compile(ProviderCompatSchema).Default({}); - expect(defaults).not.toHaveProperty("supportsDeveloperRole"); - expect(defaults).not.toHaveProperty("sendSessionAffinityHeaders"); - expect(defaults).not.toHaveProperty("supportsLongCacheRetention"); - expect(defaults).not.toHaveProperty("supportsStrictMode"); + const providerDefaults = defaults(ProviderCompatSchema); + expect(providerDefaults).not.toHaveProperty("supportsDeveloperRole"); + expect(providerDefaults).not.toHaveProperty("sendSessionAffinityHeaders"); + expect(providerDefaults).not.toHaveProperty("supportsLongCacheRetention"); + expect(providerDefaults).not.toHaveProperty("supportsStrictMode"); }); }); diff --git a/packages/ai/test/faux-provider.test.ts b/packages/ai/test/faux-provider.test.ts index ad23ed938..afdf685b6 100644 --- a/packages/ai/test/faux-provider.test.ts +++ b/packages/ai/test/faux-provider.test.ts @@ -1,3 +1,4 @@ +import { Type } from "typebox"; import { afterEach, describe, expect, it } from "vitest"; import { complete, @@ -7,7 +8,6 @@ import { fauxToolCall, registerFauxProvider, stream, - Type, } from "../src/compat.ts"; import type { AssistantMessageEvent, Context } from "../src/types.ts"; diff --git a/packages/ai/test/interleaved-thinking.test.ts b/packages/ai/test/interleaved-thinking.test.ts index b4da12835..b787a2cf7 100644 --- a/packages/ai/test/interleaved-thinking.test.ts +++ b/packages/ai/test/interleaved-thinking.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { completeSimple, getModel } from "../src/compat.ts"; import { getEnvApiKey } from "../src/env-api-keys.ts"; import type { Api, Context, Model, StopReason, Tool, ToolCall, ToolResultMessage } from "../src/types.ts"; -import { StringEnum } from "../src/utils/typebox-helpers.ts"; +import { StringEnum } from "../src/utils/string-enum.ts"; import { hasBedrockCredentials } from "./bedrock-utils.ts"; const calculatorSchema = Type.Object({ diff --git a/packages/ai/test/stream.test.ts b/packages/ai/test/stream.test.ts index 6cac98a15..3cc02268b 100644 --- a/packages/ai/test/stream.test.ts +++ b/packages/ai/test/stream.test.ts @@ -9,7 +9,7 @@ import type { Api, Context, ImageContent, Model, StreamOptions, Tool, ToolResult type StreamOptionsWithExtras = StreamOptions & Record; -import { StringEnum } from "../src/utils/typebox-helpers.ts"; +import { StringEnum } from "../src/utils/string-enum.ts"; import { hasAzureOpenAICredentials, resolveAzureDeploymentName } from "./azure-utils.ts"; import { hasBedrockCredentials } from "./bedrock-utils.ts"; import { hasCloudflareAiGatewayCredentials, hasCloudflareWorkersAICredentials } from "./cloudflare-utils.ts"; diff --git a/packages/ai/test/validation-probes.test.ts b/packages/ai/test/validation-probes.test.ts new file mode 100644 index 000000000..0ddfe8d12 --- /dev/null +++ b/packages/ai/test/validation-probes.test.ts @@ -0,0 +1,779 @@ +// The reviewers' probes of the schema switch (round 1) as regression tests: TypeBox-built and plain JSON +// (MCP-shaped) tool schemas with arguments through validateToolArguments. Each case's outcome is "ok ", "REJECT " or "THROW ". Cases marked as TypeBox-only record the documented +// differences from TypeBox (see the CHANGELOG): Refine and BigInt. + +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: TypeBox template literal syntax. +// biome-ignore-all lint/suspicious/noThenProperty: JSON Schema's `then` keyword. +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import type { Tool, ToolCall } from "../src/types.ts"; +import { validateToolArguments } from "../src/utils/validation.ts"; + +/** The outcome of one call, in the probes' format. */ +function outcome(name: string, schema: object, args: unknown): string { + const tool = { name, description: "", parameters: schema } as Tool; + const call = { type: "toolCall", id: "1", name, arguments: args } as ToolCall; + try { + return `ok ${JSON.stringify(validateToolArguments(tool, call), (_key, value) => (typeof value === "bigint" ? `${value}n` : value))}`; + } catch (error) { + const message = (error as Error).message; + return message.startsWith("Validation failed") + ? `REJECT ${message.split("\n")[1]?.trim()}` + : `THROW ${message.split("\n")[0]}`; + } +} + +type Case = { name: string; schema: object; args: unknown[] }; + +const Node = Type.Cyclic( + { Node: Type.Object({ name: Type.String(), children: Type.Array(Type.Ref("Node")) }) }, + "Node", +); + +const cases: Case[] = [ + { + name: "cyclic", + schema: Type.Object({ tree: Node }), + args: [{ tree: { name: "a", children: [{ name: "b", children: [] }] } }, { tree: { name: 1, children: [] } }], + }, + { name: "ref-with-$id", schema: Type.Object({ a: Type.Ref("A") }), args: [{ a: { x: "1" } }] }, + { + name: "function", + schema: Type.Object({ f: Type.Optional(Type.Function([], Type.Void())) }), + args: [{}, { f: "x" }], + }, + { + name: "codec", + schema: Type.Object({ + c: Type.Codec(Type.String()) + .Decode((s) => Number(s)) + .Encode((n) => String(n)), + }), + args: [{ c: "12" }, { c: 12 }], + }, + { name: "undefined-optional", schema: Type.Object({ u: Type.Optional(Type.Undefined()) }), args: [{}, { u: null }] }, + { name: "bigint", schema: Type.Object({ n: Type.BigInt() }), args: [{ n: 1 }, { n: "12" }] }, + { + name: "refine", + schema: Type.Object({ n: Type.Refine(Type.Number(), (v) => v > 0) }), + args: [{ n: 5 }, { n: -5 }], + }, + { + name: "template-literal", + schema: Type.Object({ t: Type.TemplateLiteral("id-${number}") }), + args: [{ t: "id-12" }, { t: "id-x" }, { t: 3 }], + }, + { + name: "record-string", + schema: Type.Object({ r: Type.Record(Type.String(), Type.Number()) }), + args: [{ r: { a: "1" } }, { r: { a: 1, b: "x" } }], + }, + { + name: "record-literal-keys", + schema: Type.Object({ r: Type.Record(Type.Union([Type.Literal("a"), Type.Literal("b")]), Type.Number()) }), + args: [{ r: { a: 1, b: 2 } }, { r: { a: 1 } }, { r: { a: 1, b: 2, c: 3 } }], + }, + { + name: "intersect", + schema: Type.Intersect([Type.Object({ a: Type.Number() }), Type.Object({ b: Type.Boolean() })]), + args: [ + { a: "1", b: "true" }, + { a: 1, b: true, c: 1 }, + ], + }, + { + name: "union-objects", + schema: Type.Object({ + v: Type.Union([ + Type.Object({ k: Type.Literal("x"), n: Type.Number() }), + Type.Object({ k: Type.Literal("y"), s: Type.String() }), + ]), + }), + args: [{ v: { k: "x", n: "2" } }, { v: { k: "y", s: 2 } }], + }, + { name: "minLength-astral", schema: Type.Object({ s: Type.String({ minLength: 2 }) }), args: [{ s: "😀" }] }, + { name: "maxLength-astral", schema: Type.Object({ s: Type.String({ maxLength: 1 }) }), args: [{ s: "😀" }] }, + { name: "pattern-astral", schema: Type.Object({ s: Type.String({ pattern: "^.$" }) }), args: [{ s: "😀" }] }, + { + name: "pattern-class-dash", + schema: Type.Object({ s: Type.String({ pattern: "^[\\w-]+$" }) }), + args: [{ s: "a-b" }], + }, + { + name: "pattern-unicode-prop", + schema: Type.Object({ s: Type.String({ pattern: "^\\p{L}+$" }) }), + args: [{ s: "äb" }, { s: "p{L}" }], + }, + { + name: "multipleOf-float", + schema: Type.Object({ n: Type.Number({ multipleOf: 0.01 }) }), + args: [{ n: 0.07 }, { n: 1.1 }], + }, + { + name: "uniqueItems-keyorder", + schema: Type.Object({ a: Type.Array(Type.Unknown(), { uniqueItems: true }) }), + args: [ + { + a: [ + { x: 1, y: 2 }, + { y: 2, x: 1 }, + ], + }, + { a: [1, 1.0] }, + ], + }, + { name: "const-object", schema: Type.Object({ c: Type.Literal("a") }), args: [{ c: "a" }] }, + { + name: "formats", + schema: Type.Object({ + e: Type.Optional(Type.String({ format: "email" })), + u: Type.Optional(Type.String({ format: "uri" })), + d: Type.Optional(Type.String({ format: "date-time" })), + x: Type.Optional(Type.String({ format: "made-up" })), + h: Type.Optional(Type.String({ format: "hostname" })), + r: Type.Optional(Type.String({ format: "regex" })), + i: Type.Optional(Type.String({ format: "uuid" })), + ip: Type.Optional(Type.String({ format: "ipv4" })), + }), + args: [ + { e: "a@b.co" }, + { e: "not-an-email" }, + { u: "relative/path" }, + { u: "https://x.y/z" }, + { d: "2024-01-01T00:00:00Z" }, + { d: "2024-01-01" }, + { x: "anything" }, + { h: "under_score.example" }, + { r: "[" }, + { i: "00000000-0000-0000-0000-000000000000" }, + { i: "not" }, + { ip: "999.1.1.1" }, + ], + }, + { name: "integer-big", schema: Type.Object({ n: Type.Integer() }), args: [{ n: 1e21 }, { n: 2 ** 53 + 2 }] }, + { + name: "additional-false", + schema: Type.Object({ a: Type.String() }, { additionalProperties: false }), + args: [{ a: "x", b: 1 }], + }, + { + name: "prototype-key", + schema: Type.Object({ constructor: Type.Optional(Type.String()), toString: Type.Optional(Type.Number()) }), + args: [{}, { constructor: 1 }], + }, + { + name: "proto-key-value", + schema: Type.Object({}, { additionalProperties: Type.Array(Type.String()) }), + args: [JSON.parse('{"__proto__": "x", "a": "y"}')], + }, + // Plain JSON (MCP-shaped) + { + name: "mcp-draft7-ref-definitions", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { t: { $ref: "#/definitions/T" } }, + required: ["t"], + definitions: { T: { type: "object", properties: { path: { type: "string" } }, required: ["path"] } }, + }, + args: [{ t: { path: "a" } }, { t: { path: 42 } }], + }, + { + name: "mcp-ref-unresolved", + schema: { type: "object", properties: { t: { $ref: "#/definitions/Missing" } } }, + args: [{ t: 1 }, {}], + }, + { + name: "mcp-ref-external", + schema: { type: "object", properties: { t: { $ref: "https://example.com/schemas/t.json" } } }, + args: [{ t: 1 }], + }, + { + name: "mcp-nullable-openapi", + schema: { type: "object", properties: { s: { type: "string", nullable: true } }, required: ["s"] }, + args: [{ s: null }, { s: "x" }], + }, + { + name: "mcp-draft7-tuple", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { p: { type: "array", items: [{ type: "number" }, { type: "string" }], additionalItems: false } }, + }, + args: [{ p: [1, "a"] }, { p: [1, "a", 3] }, { p: ["1", 2] }], + }, + { + name: "mcp-draft7-ref-sibling", + schema: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { s: { $ref: "#/definitions/S", maxLength: 1 } }, + definitions: { S: { type: "string" } }, + }, + args: [{ s: "abc" }], + }, + { + name: "mcp-draft4-exclusive-bool", + schema: { + $schema: "http://json-schema.org/draft-04/schema#", + type: "object", + properties: { n: { type: "number", minimum: 0, exclusiveMinimum: true } }, + }, + args: [{ n: 0 }, { n: 1 }], + }, + { + name: "mcp-id-draft4", + schema: { + $schema: "http://json-schema.org/draft-04/schema#", + id: "http://x/y", + type: "object", + properties: { a: { type: "string" } }, + }, + args: [{ a: "x" }], + }, + { + name: "mcp-bad-pattern", + schema: { type: "object", properties: { s: { type: "string", pattern: "(? `v${i}`) } }, + }, + args: [{ e: "v4999" }, { e: "v5000" }], + }, +]; + +const o = (properties: Record, extra: Record = {}) => ({ + type: "object", + properties, + ...extra, +}); +const keywordCases: [string, object, unknown[]][] = [ + [ + "oneOf-overlap", + o({ v: { oneOf: [{ type: "string" }, { type: "string", maxLength: 5 }] } }), + [{ v: "abc" }, { v: "abcdefgh" }], + ], + ["oneOf-number-integer", o({ v: { oneOf: [{ type: "number" }, { type: "integer" }] } }), [{ v: 2 }, { v: 2.5 }]], + [ + "oneOf-objects-open", + o({ + v: { + oneOf: [ + { type: "object", properties: { a: { type: "string" } } }, + { type: "object", properties: { b: { type: "string" } } }, + ], + }, + }), + [{ v: { a: "x" } }, { v: {} }], + ], + [ + "if-then-else", + o( + { kind: { type: "string" }, n: { type: "number" } }, + { if: { properties: { kind: { const: "a" } } }, then: { required: ["n"] } }, + ), + [{ kind: "a" }, { kind: "b" }], + ], + [ + "dependentRequired", + o({ a: { type: "string" }, b: { type: "string" } }, { dependentRequired: { a: ["b"] } }), + [{ a: "x" }], + ], + [ + "dependencies-d7", + o({ a: { type: "string" }, b: { type: "string" } }, { dependencies: { a: ["b"] } }), + [{ a: "x" }], + ], + ["propertyNames", o({}, { propertyNames: { pattern: "^[a-z]+$" } }), [{ abc: 1 }, { ABC: 1 }]], + ["contains", o({ l: { type: "array", contains: { type: "number" } } }), [{ l: ["a"] }, { l: ["a", 1] }]], + ["not", o({ s: { not: { type: "null" } } }), [{ s: "x" }, { s: null }]], + ["minProperties", o({ m: { type: "object", minProperties: 1 } }), [{ m: {} }]], + [ + "unevaluatedProperties", + o({ a: { type: "string" } }, { unevaluatedProperties: false }), + [{ a: "x" }, { a: "x", b: 1 }], + ], + [ + "allOf-additional-false", + { allOf: [o({ a: { type: "string" } }, { additionalProperties: false }), o({ b: { type: "string" } })] }, + [{ a: "x" }, { a: "x", b: "y" }], + ], + ["const-object", o({ c: { const: { x: 1, y: 2 } } }), [{ c: { y: 2, x: 1 } }]], + ["enum-object", o({ e: { enum: [{ x: 1, y: 2 }] } }), [{ e: { y: 2, x: 1 } }]], + ["type-array-multi", o({ v: { type: ["string", "number"] } }), [{ v: true }, { v: null }]], + ["nullable-type-array", o({ v: { type: ["string", "null"] } }, { required: ["v"] }), [{ v: null }, { v: 5 }]], + ["exclusiveMinimum-number", o({ n: { type: "number", exclusiveMinimum: 0 } }), [{ n: 0 }, { n: "0" }]], + ["maxItems", o({ l: { type: "array", maxItems: 1 } }), [{ l: [1, 2] }, { l: "x" }]], + [ + "discriminator-openapi", + o({ + p: { + oneOf: [ + o({ kind: { type: "string", enum: ["a"] }, x: { type: "number" } }, { required: ["kind"] }), + o({ kind: { type: "string", enum: ["b"] } }, { required: ["kind"] }), + ], + discriminator: { propertyName: "kind" }, + }, + }), + [{ p: { kind: "a", x: "1" } }, { p: { kind: "b" } }], + ], + [ + "ref-root-defs", + { $ref: "#/$defs/Root", $defs: { Root: o({ a: { type: "string" } }, { required: ["a"] }) } }, + [{ a: "x" }, { a: 1 }, {}], + ], + [ + "ref-recursive", + { + $defs: { N: o({ c: { type: "array", items: { $ref: "#/$defs/N" } }, v: { type: "integer" } }) }, + ...o({ root: { $ref: "#/$defs/N" } }), + }, + [{ root: { v: "2", c: [{ v: "3", c: [] }] } }], + ], + [ + "ref-percent", + { ...o({ a: { $ref: "#/$defs/a%20b" } }), $defs: { "a b": { type: "string" } } }, + [{ a: "x" }, { a: 1 }], + ], + ["ref-tilde", { ...o({ a: { $ref: "#/$defs/a~1b" } }), $defs: { "a/b": { type: "string" } } }, [{ a: "x" }]], + [ + "defs-with-$id", + { ...o({ a: { $ref: "item.json" } }), $defs: { item: { $id: "item.json", type: "string" } } }, + [{ a: "x" }, { a: 2 }], + ], + [ + "anyOf-null-default", + o({ v: { anyOf: [{ type: "string" }, { type: "null" }], default: null } }), + [{ v: null }, {}], + ], + [ + "format-date-time-lower", + o({ d: { type: "string", format: "date-time" } }), + [{ d: "2024-01-01t00:00:00z" }, { d: "2024-01-01 00:00:00Z" }], + ], + ["format-url", o({ u: { type: "string", format: "url" } }), [{ u: "not a url" }]], + ["integer-as-float-1e0", o({ n: { type: "integer" } }), [{ n: 1e3 }]], + ["required-missing-props", { type: "object", required: ["a"] }, [{}, { a: null }]], + ["string-to-number-trim", o({ n: { type: "number" } }), [{ n: " 12 " }, { n: "0x10" }, { n: "1e3" }, { n: "" }]], +]; + +const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, +}; +const json = { + type: "object", + properties: { data: { $ref: "#/$defs/Json" } }, + required: ["data"], + $defs: { + Json: { + anyOf: [ + { type: "string" }, + { type: "number" }, + { type: "boolean" }, + { type: "array", items: { $ref: "#/$defs/Json" } }, + { type: "object", additionalProperties: { $ref: "#/$defs/Json" } }, + ], + }, + }, +}; +const filter = { + type: "object", + properties: { where: { $ref: "#/$defs/F" } }, + $defs: { + F: { + oneOf: [ + { + type: "object", + properties: { field: { type: "string" }, eq: { type: "string" } }, + required: ["field", "eq"], + additionalProperties: false, + }, + { + type: "object", + properties: { and: { type: "array", items: { $ref: "#/$defs/F" } } }, + required: ["and"], + additionalProperties: false, + }, + ], + }, + }, +}; +const recursionCases: [string, object, unknown][] = [ + ["tree valid", tree, { t: ["a", ["b"]] }], + ["tree number", tree, { t: 5 }], + ["tree [5]", tree, { t: [5] }], + ["json valid", json, { data: { a: [1, "x", { b: true }] } }], + ["json null", json, { data: null }], + ["json nested null", json, { data: { a: [1, null] } }], + ["filter valid", filter, { where: { and: [{ field: "a", eq: "b" }] } }], + ["filter bad leaf", filter, { where: { and: [{ field: "a", eq: 1, x: 2 }] } }], + ["filter deep bad", filter, { where: { and: [{ and: [{ and: [{ and: [{ and: [{ field: 1 }] }] }] }] }] } }], +]; + +const expected: Record = { + 'cyclic {"tree":{"name":"a","children":[{"name":"b","children":[]}]}}': + 'ok {"tree":{"name":"a","children":[{"name":"b","children":[]}]}}', + 'cyclic {"tree":{"name":1,"children":[]}}': 'ok {"tree":{"name":"1","children":[]}}', + 'ref-with-$id {"a":{"x":"1"}}': "REJECT - a: unresolved reference A", + "function {}": "ok {}", + 'function {"f":"x"}': "REJECT - f: expected function, got string", + 'codec {"c":"12"}': 'ok {"c":"12"}', + 'codec {"c":12}': 'ok {"c":"12"}', + "undefined-optional {}": "ok {}", + 'undefined-optional {"u":null}': "ok {}", + 'bigint {"n":1}': "REJECT - n: expected bigint, got integer", + 'bigint {"n":"12"}': "REJECT - n: expected bigint, got string", + 'refine {"n":5}': 'ok {"n":5}', + 'refine {"n":-5}': 'ok {"n":-5}', + 'template-literal {"t":"id-12"}': 'ok {"t":"id-12"}', + 'template-literal {"t":"id-x"}': "REJECT - t: expected to match ^id--?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + 'template-literal {"t":3}': "REJECT - t: expected to match ^id--?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$", + 'record-string {"r":{"a":"1"}}': 'ok {"r":{"a":1}}', + 'record-string {"r":{"a":1,"b":"x"}}': "REJECT - r.b: expected number, got string", + 'record-literal-keys {"r":{"a":1,"b":2}}': 'ok {"r":{"a":1,"b":2}}', + 'record-literal-keys {"r":{"a":1}}': "REJECT - r.b: missing", + 'record-literal-keys {"r":{"a":1,"b":2,"c":3}}': 'ok {"r":{"a":1,"b":2,"c":3}}', + 'intersect {"a":"1","b":"true"}': 'ok {"a":1,"b":true}', + 'intersect {"a":1,"b":true,"c":1}': 'ok {"a":1,"b":true,"c":1}', + 'union-objects {"v":{"k":"x","n":"2"}}': 'ok {"v":{"k":"x","n":2}}', + 'union-objects {"v":{"k":"y","s":2}}': 'ok {"v":{"k":"y","s":"2"}}', + 'minLength-astral {"s":"😀"}': "REJECT - s: expected at least 2 characters", + 'maxLength-astral {"s":"😀"}': 'ok {"s":"😀"}', + 'pattern-astral {"s":"😀"}': 'ok {"s":"😀"}', + 'pattern-class-dash {"s":"a-b"}': 'ok {"s":"a-b"}', + 'pattern-unicode-prop {"s":"äb"}': 'ok {"s":"äb"}', + 'pattern-unicode-prop {"s":"p{L}"}': "REJECT - s: expected to match ^\\p{L}+$", + 'multipleOf-float {"n":0.07}': 'ok {"n":0.07}', + 'multipleOf-float {"n":1.1}': 'ok {"n":1.1}', + 'uniqueItems-keyorder {"a":[{"x":1,"y":2},{"y":2,"x":1}]}': "REJECT - a: expected unique items", + 'uniqueItems-keyorder {"a":[1,1]}': "REJECT - a: expected unique items", + 'const-object {"c":"a"}': 'ok {"c":"a"}', + 'formats {"e":"a@b.co"}': 'ok {"e":"a@b.co"}', + 'formats {"e":"not-an-email"}': "REJECT - e: expected format email", + 'formats {"u":"relative/path"}': "REJECT - u: expected format uri", + 'formats {"u":"https://x.y/z"}': 'ok {"u":"https://x.y/z"}', + 'formats {"d":"2024-01-01T00:00:00Z"}': 'ok {"d":"2024-01-01T00:00:00Z"}', + 'formats {"d":"2024-01-01"}': "REJECT - d: expected format date-time", + 'formats {"x":"anything"}': 'ok {"x":"anything"}', + 'formats {"h":"under_score.example"}': "REJECT - h: expected format hostname", + 'formats {"r":"["}': "REJECT - r: expected format regex", + 'formats {"i":"00000000-0000-0000-0000-000000000000"}': 'ok {"i":"00000000-0000-0000-0000-000000000000"}', + 'formats {"i":"not"}': "REJECT - i: expected format uuid", + 'formats {"ip":"999.1.1.1"}': "REJECT - ip: expected format ipv4", + 'integer-big {"n":1e+21}': 'ok {"n":1e+21}', + 'integer-big {"n":9007199254740994}': 'ok {"n":9007199254740994}', + 'additional-false {"a":"x","b":1}': "REJECT - b: unknown field", + "prototype-key {}": "ok {}", + 'prototype-key {"constructor":1}': 'ok {"constructor":"1"}', + 'proto-key-value {"__proto__":"x","a":"y"}': 'ok {"__proto__":["x"],"a":["y"]}', + 'mcp-draft7-ref-definitions {"t":{"path":"a"}}': 'ok {"t":{"path":"a"}}', + 'mcp-draft7-ref-definitions {"t":{"path":42}}': 'ok {"t":{"path":"42"}}', + 'mcp-ref-unresolved {"t":1}': "REJECT - t: unresolved reference #/definitions/Missing", + "mcp-ref-unresolved {}": "ok {}", + 'mcp-ref-external {"t":1}': "REJECT - t: unresolved reference https://example.com/schemas/t.json", + 'mcp-nullable-openapi {"s":null}': 'ok {"s":"null"}', + 'mcp-nullable-openapi {"s":"x"}': 'ok {"s":"x"}', + 'mcp-draft7-tuple {"p":[1,"a"]}': 'ok {"p":[1,"a"]}', + 'mcp-draft7-tuple {"p":[1,"a",3]}': "REJECT - p.2: no value is allowed here", + 'mcp-draft7-tuple {"p":["1",2]}': 'ok {"p":[1,"2"]}', + 'mcp-draft7-ref-sibling {"s":"abc"}': "REJECT - s: expected at most 1 characters", + 'mcp-draft4-exclusive-bool {"n":0}': 'ok {"n":0}', + 'mcp-draft4-exclusive-bool {"n":1}': 'ok {"n":1}', + 'mcp-id-draft4 {"a":"x"}': 'ok {"a":"x"}', + 'mcp-bad-pattern {"s":"y"}': 'ok {"s":"y"}', + 'mcp-invalid-regex {"s":"y"}': "THROW Invalid regular expression: /[/: Unterminated character class", + 'mcp-type-integer-float-0 {"n":1}': 'ok {"n":1}', + 'mcp-type-integer-float-0 {"n":1.5}': 'ok {"n":1}', + 'mcp-format-unknown {"s":"xx"}': 'ok {"s":"xx"}', + 'mcp-required-null-string {"q":null}': 'ok {"q":"null"}', + 'mcp-array-json-string {"p":"[\\"a\\",\\"b\\"]"}': 'ok {"p":["[\\"a\\",\\"b\\"]"]}', + 'mcp-integer-float {"n":3.7}': 'ok {"n":3}', + 'mcp-integer-float {"n":"3.7"}': 'ok {"n":3}', + 'mcp-empty-schema {"a":1}': 'ok {"a":1}', + 'mcp-type-object-no-props {"a":1}': 'ok {"a":1}', + 'mcp-boolean-true-prop {"a":1}': 'ok {"a":1}', + 'mcp-boolean-true-prop {"b":1}': "REJECT - b: no value is allowed here", + 'mcp-huge-enum {"e":"v4999"}': 'ok {"e":"v4999"}', + 'mcp-huge-enum {"e":"v5000"}': + 'REJECT - e: expected one of ["v0","v1","v2","v3","v4","v5","v6","v7","v8","v9","v10","v11","v12","v13","v14","v15","v16","v17","v18","v19", ... and 4980 more]', + 'oneOf-overlap {"v":"abc"}': "REJECT - v: matches more than one alternative", + 'oneOf-overlap {"v":"abcdefgh"}': 'ok {"v":"abcdefgh"}', + 'oneOf-number-integer {"v":2}': "REJECT - v: matches more than one alternative", + 'oneOf-number-integer {"v":2.5}': 'ok {"v":2.5}', + 'oneOf-objects-open {"v":{"a":"x"}}': "REJECT - v: matches more than one alternative", + 'oneOf-objects-open {"v":{}}': "REJECT - v: matches more than one alternative", + 'if-then-else {"kind":"a"}': "REJECT - n: missing", + 'if-then-else {"kind":"b"}': 'ok {"kind":"b"}', + 'dependentRequired {"a":"x"}': "REJECT - b: required when a is present", + 'dependencies-d7 {"a":"x"}': "REJECT - b: required when a is present", + 'propertyNames {"abc":1}': 'ok {"abc":1}', + 'propertyNames {"ABC":1}': "REJECT - ABC: invalid property name", + 'contains {"l":["a"]}': "REJECT - l: expected at least 1 matching item", + 'contains {"l":["a",1]}': 'ok {"l":["a",1]}', + 'not {"s":"x"}': 'ok {"s":"x"}', + 'not {"s":null}': "ok {}", + 'minProperties {"m":{}}': "REJECT - m: expected at least 1 properties", + 'unevaluatedProperties {"a":"x"}': 'ok {"a":"x"}', + 'unevaluatedProperties {"a":"x","b":1}': "REJECT - b: unknown field", + 'allOf-additional-false {"a":"x"}': 'ok {"a":"x"}', + 'allOf-additional-false {"a":"x","b":"y"}': "REJECT - b: unknown field", + 'const-object {"c":{"y":2,"x":1}}': 'ok {"c":{"y":2,"x":1}}', + 'enum-object {"e":{"y":2,"x":1}}': 'ok {"e":{"y":2,"x":1}}', + 'type-array-multi {"v":true}': 'ok {"v":"true"}', + 'type-array-multi {"v":null}': "ok {}", + 'nullable-type-array {"v":null}': 'ok {"v":null}', + 'nullable-type-array {"v":5}': 'ok {"v":"5"}', + 'exclusiveMinimum-number {"n":0}': "REJECT - n: expected > 0", + 'exclusiveMinimum-number {"n":"0"}': "REJECT - n: expected > 0", + 'maxItems {"l":[1,2]}': "REJECT - l: expected at most 1 items", + 'maxItems {"l":"x"}': 'ok {"l":["x"]}', + 'discriminator-openapi {"p":{"kind":"a","x":"1"}}': 'ok {"p":{"kind":"a","x":1}}', + 'discriminator-openapi {"p":{"kind":"b"}}': 'ok {"p":{"kind":"b"}}', + 'ref-root-defs {"a":"x"}': 'ok {"a":"x"}', + 'ref-root-defs {"a":1}': 'ok {"a":"1"}', + "ref-root-defs {}": "REJECT - a: missing", + 'ref-recursive {"root":{"v":"2","c":[{"v":"3","c":[]}]}}': 'ok {"root":{"v":2,"c":[{"v":3,"c":[]}]}}', + 'ref-percent {"a":"x"}': 'ok {"a":"x"}', + 'ref-percent {"a":1}': 'ok {"a":"1"}', + 'ref-tilde {"a":"x"}': 'ok {"a":"x"}', + 'defs-with-$id {"a":"x"}': 'ok {"a":"x"}', + 'defs-with-$id {"a":2}': 'ok {"a":"2"}', + 'anyOf-null-default {"v":null}': 'ok {"v":null}', + "anyOf-null-default {}": "ok {}", + 'format-date-time-lower {"d":"2024-01-01t00:00:00z"}': 'ok {"d":"2024-01-01t00:00:00z"}', + 'format-date-time-lower {"d":"2024-01-01 00:00:00Z"}': "REJECT - d: expected format date-time", + 'format-url {"u":"not a url"}': "REJECT - u: expected format url", + 'integer-as-float-1e0 {"n":1000}': 'ok {"n":1000}', + "required-missing-props {}": "REJECT - a: missing", + 'required-missing-props {"a":null}': 'ok {"a":null}', + 'string-to-number-trim {"n":" 12 "}': 'ok {"n":12}', + 'string-to-number-trim {"n":"0x10"}': 'ok {"n":16}', + 'string-to-number-trim {"n":"1e3"}': 'ok {"n":1000}', + 'string-to-number-trim {"n":""}': 'ok {"n":0}', + "tree valid": 'ok {"t":["a",["b"]]}', + "tree number": 'ok {"t":"5"}', + "tree [5]": 'ok {"t":["5"]}', + "json valid": 'ok {"data":{"a":[1,"x",{"b":true}]}}', + "json null": 'ok {"data":"null"}', + "json nested null": 'ok {"data":{"a":[1,"null"]}}', + "filter valid": 'ok {"where":{"and":[{"field":"a","eq":"b"}]}}', + "filter bad leaf": "REJECT - where.and.0.eq: expected string, got integer", + "filter deep bad": "REJECT - where.and.0.and.0.and.0.and.0.and.0.eq: missing", +}; + +/** Every case with the key of its expected outcome: the name and the arguments, or the name alone for recursion. */ +const all: Array<[string, string, object, unknown]> = [ + ...cases.flatMap(({ name, schema, args }) => + args.map((arg): [string, string, object, unknown] => [`${name} ${JSON.stringify(arg)}`, name, schema, arg]), + ), + ...keywordCases.flatMap(([name, schema, args]) => + args.map((arg): [string, string, object, unknown] => [`${name} ${JSON.stringify(arg)}`, name, schema, arg]), + ), + ...recursionCases.map(([name, schema, arg]): [string, string, object, unknown] => [name, name, schema, arg]), +]; + +describe("reviewer probes", () => { + for (const [key, name, schema, arg] of all) { + it(key, () => { + expect(outcome(name, schema, arg)).toBe(expected[key]); + }); + } +}); + +describe("reviewer probes (Sol)", () => { + const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, + }; + const solCases: Array<[string, object, unknown, string]> = [ + // TypeBox-only: Type.BigInt and Type.Undefined have no JSON form; TypeBox converted to them. + [ + "BigInt from a JSON string", + Type.Object({ n: Type.BigInt() }), + { n: "42" }, + "REJECT - n: expected bigint, got string", + ], + [ + "Undefined from JSON null", + Type.Object({ n: Type.Undefined() }), + { n: null }, + "REJECT - n: expected undefined, got null", + ], + // TypeBox-only: its additionalProperties also converted declared properties; pi-schema converts only undeclared ones. + [ + "additionalProperties on declared properties", + Type.Object({ amount: Type.Number(), label: Type.String() }, { additionalProperties: Type.Integer() }), + { amount: 3.5, label: "7" }, + 'ok {"amount":3.5,"label":"7"}', + ], + // TypeBox-only: property names are names, not regular expressions. + [ + "property names are not patterns", + Type.Object({ "a.b": Type.Integer(), axb: Type.String() }), + { "a.b": "4", axb: "5.8" }, + 'ok {"a.b":4,"axb":"5.8"}', + ], + // TypeBox-only: Refine runs only in TypeBox's own validator. + [ + "Refine is not run", + Type.Object({ mode: Type.Refine(Type.String(), (value) => value === "safe") }), + { mode: "unsafe" }, + 'ok {"mode":"unsafe"}', + ], + [ + "scalar-or-array that refers to itself", + { + type: "object", + properties: { + value: { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#/properties/value" } }] }, + }, + required: ["value"], + }, + { value: "42" }, + 'ok {"value":42}', + ], + [ + "StringEnum with an empty member from null", + Type.Object({ mode: Type.Unsafe({ type: "string", enum: ["", "x"] }) }), + { mode: null }, + 'ok {"mode":""}', + ], + [ + "Cyclic with named local definitions", + Type.Object({ + node: Type.Cyclic( + { Node: Type.Object({ n: Type.Number(), next: Type.Optional(Type.Ref("Node")) }) }, + "Node", + ), + }), + { node: { n: 1 } }, + 'ok {"node":{"n":1}}', + ], + [ + "unpaired high surrogate, minLength 2", + Type.Object({ s: Type.String({ minLength: 2 }) }), + { s: "\ud800A" }, + 'ok {"s":"\\ud800A"}', + ], + [ + "unpaired high surrogate, maxLength 1", + Type.Object({ s: Type.String({ maxLength: 1 }) }), + { s: "\ud800A" }, + "REJECT - s: expected at most 1 characters", + ], + ["integer from a string", Type.Object({ n: Type.Integer() }), { n: "42" }, 'ok {"n":42}'], + ["recursive union with a value no branch converts", tree, { t: {} }, "REJECT - t: expected string, got object"], + ]; + for (const [name, schema, arg, result] of solCases) { + it(name, () => { + expect(outcome(name, schema, arg)).toBe(result); + }); + } + + it("accepts the last member of a 10,000-value enum", () => { + const values = Array.from({ length: 10_000 }, (_, i) => `member${i}`); + const schema = { type: "object", properties: { choice: { type: "string", enum: values } }, required: ["choice"] }; + expect(outcome("enum", schema, { choice: "member9999" })).toBe('ok {"choice":"member9999"}'); + expect(outcome("enum", schema, { choice: "x" })).toBe( + `REJECT - choice: expected one of ${JSON.stringify(values.slice(0, 20)).slice(0, -1)}, ... and 9980 more]`, + ); + }); + + it("keeps the validator of a schema object, as before, also when the schema is changed later", () => { + const parameters = Type.Object({ n: Type.Number({ minimum: 0 }) }); + expect(outcome("late", parameters, { n: "4" })).toBe('ok {"n":4}'); + Object.assign(parameters.properties.n, { minimum: 10 }); + expect(outcome("late", parameters, { n: "4" })).toBe('ok {"n":4}'); + }); +}); + +describe("reviewer probes (round 2)", () => { + const item = { type: "object", properties: { n: { type: "integer", minimum: 1 } } }; + const nested: Case[] = [ + { name: "declared", schema: { type: "object", properties: { entry: item } }, args: [{ entry: { n: null } }] }, + { name: "additional", schema: { type: "object", additionalProperties: item }, args: [{ entry: { n: null } }] }, + { + name: "pattern", + schema: { type: "object", patternProperties: { "^entry$": item }, additionalProperties: false }, + args: [{ entry: { n: null } }], + }, + ]; + for (const { name, schema, args } of nested) { + it(`drops an optional null inside ${name} property values`, () => { + expect(outcome(name, schema, args[0])).toBe('ok {"entry":{}}'); + }); + } + + for (const keyword of ["anyOf", "oneOf"]) { + it(`converts a ${keyword} branch together with the node's sibling properties`, () => { + const schema = { + type: "object", + [keyword]: [ + { type: "object", properties: { a: { type: "array", items: { type: "integer" } } }, required: ["a"] }, + { type: "object", properties: { b: { type: "boolean" } }, required: ["b"] }, + ], + properties: { c: { type: "integer" } }, + required: ["c"], + }; + expect(outcome(keyword, schema, { a: "3", c: "2" })).toBe('ok {"a":[3],"c":2}'); + }); + } + + it("rejects a deep value every branch of a recursive object union rejects, without exponential work", () => { + const S = { + anyOf: [ + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["a"] }, + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["b"] }, + ], + }; + const schema = { type: "object", properties: { s: { $ref: "#/$defs/S" } }, $defs: { S } }; + let value: unknown = { x: 1 }; + for (let i = 0; i < 40; i++) value = { c: value }; + expect(outcome("deep", schema, { s: value })).toBe("REJECT - s.a: missing"); + }); +}); diff --git a/packages/ai/test/validation.test.ts b/packages/ai/test/validation.test.ts index 7cebbe541..474d0b6cf 100644 --- a/packages/ai/test/validation.test.ts +++ b/packages/ai/test/validation.test.ts @@ -1,5 +1,4 @@ import { Type } from "typebox"; -import { Compile } from "typebox/compile"; import { describe, expect, it } from "vitest"; import type { JsonValue, Tool, ToolCall } from "../src/types.ts"; import { validateToolArguments } from "../src/utils/validation.ts"; @@ -61,7 +60,7 @@ describe("validateToolArguments", () => { } }); - it("coerces serialized plain JSON schemas with AJV-compatible primitive rules", () => { + it("coerces serialized plain JSON schemas like TypeBox schemas", () => { const passingCases: Array<{ schema: Tool["parameters"]; input: unknown; @@ -75,7 +74,8 @@ describe("validateToolArguments", () => { { schema: { type: "boolean" } as Tool["parameters"], input: "false", expected: false }, { schema: { type: "boolean" } as Tool["parameters"], input: 1, expected: true }, { schema: { type: "boolean" } as Tool["parameters"], input: 0, expected: false }, - { schema: { type: "string" } as Tool["parameters"], input: null, expected: "" }, + // TypeBox's conversion rules come first: null is "null" for a required string. + { schema: { type: "string" } as Tool["parameters"], input: null, expected: "null" }, { schema: { type: "string" } as Tool["parameters"], input: true, expected: "true" }, { schema: { type: "null" } as Tool["parameters"], input: "", expected: null }, { schema: { type: "null" } as Tool["parameters"], input: 0, expected: null }, @@ -90,6 +90,11 @@ describe("validateToolArguments", () => { input: "1", expected: 1, }, + { schema: { type: "boolean" } as Tool["parameters"], input: "1", expected: true }, + { schema: { type: "boolean" } as Tool["parameters"], input: "0", expected: false }, + { schema: { type: "null" } as Tool["parameters"], input: "null", expected: null }, + { schema: { type: "integer" } as Tool["parameters"], input: "42.1", expected: 42 }, + { schema: { type: "array", items: { type: "number" } } as Tool["parameters"], input: "7", expected: [7] }, ]; for (const testCase of passingCases) { @@ -184,10 +189,6 @@ describe("validateToolArguments", () => { { type: ["array", "null"], items: { type: "string" } } as Tool["parameters"], null, ); - // The CSP test above selects TypeBox's process-wide interpreted fallback, so exercise the generated validator explicitly. - const generatedCheck = new Function(Compile(tool.parameters).Code())() as (value: unknown) => boolean; - - expect(generatedCheck(toolCall.arguments)).toBe(true); expect(validateToolArguments(tool, toolCall)).toEqual({ value: null }); }); @@ -196,10 +197,10 @@ describe("validateToolArguments", () => { schema: Tool["parameters"]; input: unknown; }> = [ - { schema: { type: "boolean" } as Tool["parameters"], input: "1" }, - { schema: { type: "boolean" } as Tool["parameters"], input: "0" }, - { schema: { type: "null" } as Tool["parameters"], input: "null" }, - { schema: { type: "integer" } as Tool["parameters"], input: "42.1" }, + { schema: { type: "boolean" } as Tool["parameters"], input: "yes" }, + { schema: { type: "number" } as Tool["parameters"], input: "abc" }, + { schema: { type: "null" } as Tool["parameters"], input: "nil" }, + { schema: { type: "integer" } as Tool["parameters"], input: "4x" }, ]; for (const testCase of failingCases) { @@ -207,6 +208,25 @@ describe("validateToolArguments", () => { expect(() => validateToolArguments(tool, toolCall)).toThrow("Validation failed"); } }); + it("lists pi-schema's errors, then the arguments it received", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "number" } as Tool["parameters"], "many"); + expect(() => validateToolArguments(tool, toolCall)).toThrow( + 'Validation failed for tool "echo":\n - value: expected number, got string\n\nReceived arguments:\n{\n "value": "many"\n}', + ); + }); + + it("reports a missing required property at the property", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "string" } as Tool["parameters"], "x"); + expect(() => validateToolArguments(tool, { ...toolCall, arguments: {} })).toThrow(" - value: missing"); + }); + + it("returns a copy of the arguments, also when nothing is coerced", () => { + const { tool, toolCall } = createToolCallWithPlainSchema({ type: "string" } as Tool["parameters"], "x"); + const args = validateToolArguments(tool, toolCall); + expect(args).toEqual({ value: "x" }); + expect(args).not.toBe(toolCall.arguments); + }); + // #10897 it("reports malformed arguments JSON instead of validating empty arguments", () => { const tool: Tool = { diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 84c74692f..397107fa1 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -13,5 +13,8 @@ export default defineConfig({ }, resolve: { alias: [{ find: /^@earendil-works\/pi-telemetry$/, replacement: telemetrySrcIndex }], + // Workspace packages without a build (pi-schema) resolve to their source. + conditions: ["source"], }, + ssr: { resolve: { conditions: ["source"] } }, }); \ No newline at end of file diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index f475c4c8a..8ea8cb930 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -2,8 +2,24 @@ ## [Unreleased] +### Added + +- Chord v2's plugins component gains the plugin model (slice 12): `createPlugins({ api: { name, versions } })` checks every import of the app's versioned API (`@acme/app/v3`, per-kind modules such as `@acme/app/v3/web`): an unlisted version, another kind's module, two versions in one facet and a direct import of Chord are build errors naming the file and line. Roots may be `{ path, required }`: a required root's plugin failing at `load` rejects it with a `PluginError` after the load settled. Files named `..` (`contract.web.tsx`) are of that kind, refused in facets of another kind with the chain of imports named. Single-file plugins (`hello.server.ts` at a root's top, grouped by name, host packages only). `installPlugin({ git, ref } | { npm, version }, root, ctx)` installs plugins from git (a shallow clone of a branch, tag or commit; a repository of several plugins too) or npm; a package lists the plugin folders it provides in its `package.json` (`"chord": { "plugins": ["./notes", "plugins/*", "!plugins/draft"] }`), used by discovery and `installPlugin` (without it: one plugin when the package's top has facet files, else its subfolders with a `package.json`). `testPlugin(dir, { kind, modules, stubs, principal }, ctx)` builds a plugin as production does and loads it on an in-memory host beside stub facets, collecting its contributions to registries nobody serves and listing the contracts it waits for. Agents' guide: `docs/next/plugin-authoring.md`. +- Chord v2 gains the app kit (slice 13, `src/next-node/app`, to be exported as `@earendil-works/chord/next/app`): `startServer

({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, install?, onError }, ctx)` lays out the data directory (`server/store.sqlite`, `workers//

/`, `plugins/`, `logs/`, `owner-token`; one server per data directory), creates the server's store and host, HTTP, one workers component per worker kind and the plugins component over the app's required root and the user's root, and loads the server's plugins with the kit's objects; `startWorker({ api, kinds, busy?, onError })` is a worker's entry (its directory and store, the link to the server first, its plugins). The start-data objects `AppInfo` and `WorkerInfo` serve process values to plugins. Sessions (`Sessions`: `issue`, `verify`, `revoke`, `list`; tokens kept as SHA-256 hashes, compared in constant time; persisted ones end after 400 days unused and their browser cookie is renewed daily while used, ephemeral ones live in memory, at most `limits.ephemeralSessions`) authenticate a signed `HttpOnly`, `SameSite=Strict` cookie or `Authorization: Bearer`; a one-time owner link per start (`app.ownerLink()`, the token in the URL's fragment, single use, 10 minutes), an owner token file (mode 0600, written atomically) and device pairing (`POST /_chord/pair`, owner approval, a poll returning the token once; bounded in all and per client address) sign the owner in. The data directory, its stores and secret files are made private (0700, 0600) at every start; the home directory, a file system's root and another user's directory are refused. `Files` per host: small and chunked writes at a stored offset, ranged reads, a watchable index, caps, at most `limits.uploads` uploads under way. `app.exportData(address, path, ctx)` and `app.importData(archive, address, ctx)` move the data a worker created (its store and files) as a tar without stopping anything: a running worker writes its own store at one consistent point, else the server reads it under the worker's process lock (import refusing an existing address, another app's archive, every path leaving the target, and archives past `limits.importBytes` or `limits.importEntries`). `app.applyChanges(ctx)` and `app.plugins.install(source, ctx)`. Optional plugins as exported facets: `defineUsers(profile)`, `inviteFacet()`, `ephemeralLoginFacet()`. +- Chord v2 gains web and terminal surfaces (slice 11): `webSurface` and `tuiSurface` (`src/next-node/surfaces`) build a client kind's base bundle from the app's entry at start (content-hashed, served forever) and publish its plugins' facets as scripts that register themselves (`chordFacet("", "", function (require, module, exports) {…})`) and one combined file, with `WebScripts` and `TuiScripts` showing the current set (a rollout step publishes without waiting for clients) and the page served through the app's own template (`PageAssets`, `pageScripts`). `startPage` (`src/next-web`) runs a page's client host: the connection loop (`keepConnected`: 250 ms doubling to 5 s, wakes, silent sockets, `canRetry`), plugins applied in place on a new set (a failed one keeps its old facet), a reload when the server's base bundle changed, and the active theme as CSS variables. React bindings (`src/next-react`): `HostProvider`, `useHold`, `useWatch`, `useRegistry`, `useRefState` (React is an optional peer dependency). The terminal: `startTerminal` (`src/next-node/tui`) signs in with a token, a token file or device pairing and runs the app's terminal code always in a sandbox (a child process on pipes with Node's permission model: no file system, terminal, processes, workers, network or SQLite; Node 26.5.1 or later), filtering its output, relaying its link and confirming its actions (copy, open a URL, attach a file the user names); `startTerminalRuntime` and `TuiScope` (`src/next-tui`) inside it. Themes: the `Themes` registry, `ThemeChoice` (one dark and one light pick per principal), `followActiveTheme`, `themeVariables`, `terminalColor`. `startServer` gains `web` and `tui`. Plugin builds scope `*.module.css` files per plugin and file. Ephemeral sessions end with their client's last link (after a minute). The HTTP component's links endpoint gains an `opened` hook. +- Chord v2's `HttpRequest` gains `remoteAddress`, the peer's address as the listener knows it (for limits per client). +- Chord v2's HTTP component gains `readSignedCookie(request, name, secret)`: `readCookie`'s value with its signed expiry, for renewing a cookie before it expires. + +### Changed + +- Chord v2's HTTP component denies by default (slice 13): a request or plugin socket without a principal gets 401 before plugin code runs unless its route or socket is marked `public: true` (`{ handler, public: true }`; a `crossSite` route is public), and the `/chord` links endpoint refuses requests without a principal. +- Chord v2 plugins import each other by relative path within a root (copied into the importer's script) and across repositories by package name through the package's `exports`, pinned in `dependencies`; slice 10's imports by plugin name (`@app/notes/contract`) are removed, and a relative path leaving the root is a build error. + +## [1.5.0] - 2026-11-02 + ### Breaking Changes +- Schemas are JSON Schema checked by `@earendil-works/pi-schema` instead of TypeBox, which is no longer a dependency of Chord. Contracts, `self.state(...)` schemas and method signatures take pi-schema builders or any JSON Schema object (TypeBox-built schemas still validate, being JSON Schema, but type as `unknown`); handler and reference types come from pi-schema's `Static`. Validation, input defaults and the stripping of unknown fields work on any JSON Schema by its shape (before, defaults and stripping applied only to TypeBox-built schemas). `format` is an annotation, no longer checked: `str({ format: "email" })` accepts `"nope"` (TypeBox checked the formats it knew); check a format in the handler where it matters. An `invalid` error's message is now ` ` with pi-schema's paths and messages (a missing field at its own path, `missing`; `expected string, got number` instead of `must be string`), and its `details` are pi-schema's `ValidationError`s (`{ path, keyword, message }`) instead of TypeBox's errors. Link frames, descriptions, fingerprints and identities are unchanged for schemas with the same JSON. Migration, keeping a contract's fingerprint (it is computed from the schema's JSON): `Type.String/Number/Integer/Boolean/Null/Unknown/Array/Union/Optional(...)` become `str/num/int/bool/nul/unknown/arr/union/opt(...)` with the same options; `Type.Object(p)` becomes `obj(p, { open: true })` (pi-schema's `obj` is closed by default) and `Type.Object(p, { additionalProperties: false })` `obj(p)`; `Type.Enum(values)` becomes `oneOf(values)`; `Type.Literal(v)` becomes `{ type: typeof v, const: v }` (pi-schema's `lit(v)` has no `type`); `Type.Record(Type.String(), v)` becomes `{ type: "object", patternProperties: { "^.*$": v } }` (pi-schema's `map(v)` uses `additionalProperties`); `Type.Intersect(list)` becomes `{ allOf: list }`; `Type.Function(params, result)` and `Type.Void()` become `{ type: "function", parameters, returnType }` and `{ type: "void" }`; `Type.Unsafe(schema)` becomes `schema as Schema`; `Type.Immutable(schema)` becomes the schema (cast its `Schema` type); `opt(...)` marks only an `obj` property; `Static` and `TSchema` come from pi-schema (`Static`, `Schema`). pi-schema's own `lit`, `map` and closed `obj` give other JSON and so other fingerprints, which matters only while peers of both versions talk. - `traceIn(options, fn, context)` takes the context last (was `traceIn(context, options, fn)`), like the rest of the API. - Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. - `SqliteExecutor.runCounted(sql, ...params)` runs one statement like `run` and resolves with the rows it inserted, updated or deleted, as SQLite's `changes()` counts them (index writes and rows changed by triggers do not count). The Node driver takes it from `node:sqlite`; the Durable Object driver reads `SELECT changes()` after the statement (the cursor's `rowsWritten` also counts index writes), so `run` stays as cheap as before. Migration: custom `SqliteDatabase` and `SqliteExecutor` implementations add `runCounted`. @@ -13,7 +29,11 @@ ### Added +- Chord v2's store (`@earendil-works/chord/next`) gains `ChordStore.commitSeries(commits, receive, ctx)`: several commits in one backend transaction, each at its own seq with its own outbox row, the store metadata written once. Each commit's check runs on a step that sees the commits before it, before its writes; a check that throws refuses that commit and stores the ones before it, while a failed statement or write rolls the whole series back, and a failed `COMMIT` stops the store. It replaces `commitInBatch`, which is removed. Durable's `StoreStorage` stores each Session commit call through it. - Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). +- Chord v2 (`@earendil-works/chord/next`) gains registries, writables, wake signals and root scopes (slice 8): `defineRegistry(name, { version })` tokens shared by plugins, served by the defining plugin's facet with `host.serve(R, { compose, accepts? })`; contributions from facets (`defineFacet((host, self) => self.contribute(R, value))`) and host-lifetime objects (`StoreSelf.contribute`), plain values or Chord watchables, in a table sorted by plugin name and composed lazily (once at the end of `load`'s setups, then when read or followed, in the batch that changed the table); readers through `self.registry(R)` (a declared dependency: an object activates once the registry has a value, and fails `transient` with the new reason `unavailable/undeclared` when nobody serves it) and `host.registry(R, ctx)`, a `RegistryValue` with `observe(observer, ctx)`; older contributions and readers adapted by `accepts`; the helpers `keyedEntries` and `orderedEntries`; the `onError` kinds `registry_conflict`, `registry_failed`, `contribution_failed` and `contribution_refused`. `self.writable(initial)` is an in-process value an attempt sets (identity compared); native sources (writables, registry values, `ref.state`) are derive sources. `self.retryOn(...sources)` retries a failed object when a registry value changes. `host.root(fn, context, { principal? })` runs code outside objects with a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes. +- Chord v2 gains plugins (slice 7c): `src/next-node/plugins` (Node, to be exported as `@earendil-works/chord/next/plugins`) finds plugin directories, builds each facet with esbuild (now an optional peer dependency of `@earendil-works/chord`, imported only there), evaluates it with `vm.Script`, maps plugin stack traces (`mapStack`), installs plugin dependencies, watches plugin and base code, and rolls reloads out across hosts. `host.load` and `host.reload` work per plugin: a plugin whose setup, factory or declarations fail is left out at `load` (`{ failed }`) or kept on its old code at `reload` (`failed` in the report, `reload(facets, ctx, { failed })` for build and evaluation failures), with a closure rule keeping a consumer old when its provider was kept old; an abort after a reload's commit rejects with `details.committed`. +- Chord v2 gains HTTP and workers (slice 10): `src/next-node/http` (to be exported as `@earendil-works/chord/next/http`) builds a Web-standard `fetch` handler from the routes, wraps, sockets, hosts, relays and CSP sources plugins contribute to the registry `Http`, with authenticators from the registry `Authenticators`, responses as descriptions (`json`, `html`, `bytes` with ranges, `redirect`, `error`; `fileResponse`), and a security baseline on by default (provenance by the socket and `Host`, foreign requests anonymous, `Origin` checks, default headers with a per-response CSP nonce, signed cookies, safe downloads); Chord's own Node listener as a host-lifetime object, `nodeRequest` for apps mounting it on their own Node server, the links endpoint and plugin sockets with a heartbeat, closed at once when their contribution leaves. `ws` is an optional peer dependency, imported only there. `src/next-node/workers` (`createWorkers`, `workerMain`) runs a forked worker per address of a host kind, started on demand, stopped when idle, never restarted by itself (a failure stays until `retry` or a rollout), its plugins built by the server and shipped. The plugins component defines one definition per facet kind (`kinds: { name: { platform, hostPackages } }`, modules given at `plugins.load(host, { kind, core, modules })`), splits loading into `createPluginRuntime()` (no esbuild), offers `scripts(kind)`, rolls out with one result per host (`RolloutStep` returns `{ hosts }`), and resolves cross-plugin imports by plugin name (`@app/notes/contract`). Kernel: `host.route` returns a `RouteHandle` (`retry`, `suspend`) and a route has no backoff timer (a failed `resolve` fails its holders `waiting`); `host.root(fn, ctx, { principal: undefined })` runs anonymous; a link reads `origin` only on parent and route links (a protocol error on a child link) and drops a per-origin record once it is idle. - Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. @@ -26,11 +46,13 @@ ### Changed +- Chord v2 (`@earendil-works/chord/next`) checks its schemas with `@earendil-works/pi-schema` instead of TypeBox: its helpers (`str`, `num`, `int`, `bool`, `nul`, `lit`, `oneOf`, `union`, `obj`, `arr`, `map`, `opt`, `bytes`, `unknown`) build on pi-schema's builders and keep the JSON contracts were fingerprinted with, so every fingerprint, normal form and stored record stays the same (a test records them and reads a store written before the switch). `Static` and `Schema` (was `TSchema`) come from pi-schema; `opt(...)` marks an object field or a table column and is no schema elsewhere (a top-level `opt` had no effect before); a schema with a non-enumerable key (a TypeBox schema's metadata) is refused at declaration; `map(bytes())` is accepted now (it was refused while `bytes()` carried a TypeBox refinement). `format` is an annotation, no longer checked (`{ type: "string", format: "email" }` accepts `"nope"`; TypeBox checked the formats it knew). A failed check's message comes from pi-schema (`args/title: expected string, got integer`), its `details.path` escapes `~` and `/` (JSON Pointer); reasons and codes are unchanged. Declaring and compiling are cheaper (a 20-method contract 1.07 ms to 0.86 ms, a validator's first use 172 µs to 48 µs), a check costs the same (0.25 µs), the message envelope check up to 100 ns more per message. - The delta tracker keeps its one open change in a plain field and needs a `WeakRef` only for a second change open at the same time (one per change before). A `WeakRef` target survives the scavenges of its job, so a run of changes in one job promoted every change's overlay to old space: 200,000 small changes in one job now promote 2 MB instead of 47.5 MB and take 3.1 µs each instead of 3.8 µs (3.0 instead of 3.5 µs when they span jobs). A draft dropped without `prepare()` or `abort()` now stays referenced by its tracker until the tracker's next `beginChange()` or `adopt()`. - `applyImmutable()`, `applyImmutableBatches()` and tracker revisions copy each container on a changed path with spread (or `Object.assign()` onto a null-prototype object) instead of one `Object.defineProperty()` per key, about 25 times faster per container ([#9265](https://github.com/earendil-works/pi/issues/9265)) ### Fixed +- A host a route resolves is admitted even when the relayed reference that asked for it is released, its client disconnects, or its principal is refused or revoked while the route resolves. Before, the parent never read the new host's link: a spawned worker's own references to its parent (its `lifetime: "host"` objects' upstream references) waited until they failed `unavailable/ready_timeout`, reported as a possible reference cycle across hosts. - The Node SQLite driver keeps the 256 most recently used prepared statements per connection instead of every statement it ever prepared, so callers generating SQL of unbounded shapes (a query builder) no longer grow memory without bound. - Fixed append ops from the delta tracker and `diffRevisions()` keeping the whole string they were cut from alive (V8 represents a slice as a view of its source), so a delta chain, a watch backlog or a replica built from appends to a growing text held one copy of the text per commit ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/chord/README.md b/packages/chord/README.md index 9d9d95587..081e3fe21 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -8,7 +8,7 @@ Chord builds applications out of **objects** that live in several processes (a s - **State** is observable JSON. Remote viewers get a snapshot followed by deltas. - **Hot reload** swaps code while it runs. Work in flight finishes on the code it started with, state is kept, and incompatible changes migrate at a safe point. -Contracts are runtime values described with [TypeBox](https://github.com/sinclairzx81/typebox) schemas. Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. +Contracts are runtime values described with JSON Schema, written with the builders of `@earendil-works/pi-schema` (any JSON Schema object works; `format` is an annotation, never checked). Hosts exchange JSON frames over links: in memory, a Node IPC channel, or a WebSocket. ## Entry points @@ -27,18 +27,18 @@ Contracts are runtime values described with [TypeBox](https://github.com/sinclai ## Quick start ```ts -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createHost, hostNamed, method, remote } from "@earendil-works/chord"; // contract.ts: shared by provider and consumers export const Counter = remote("demo.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }), host: hostNamed("server"), // where its objects live; callers never name a host variants: { default: { - state: { count: Type.Number() }, - methods: { increment: method(Type.Object({ by: Type.Number() })) }, + state: { count: num() }, + methods: { increment: method(obj({ by: num() })) }, }, }, }); @@ -46,7 +46,7 @@ export const Counter = remote("demo.counter", { const host = createHost({ name: "server", self: { kind: "server" } }); host.serve(Counter, ({ name }, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { variant: "default", count, diff --git a/packages/chord/docs/guide.md b/packages/chord/docs/guide.md index 59c1414b7..c3b7b3211 100644 --- a/packages/chord/docs/guide.md +++ b/packages/chord/docs/guide.md @@ -36,17 +36,17 @@ Chord builds applications out of **objects** that live in several processes (a s ## Quick start ```ts -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createHost, method, remote } from "@earendil-works/chord"; // contract.ts: shared by provider and consumers export const Counter = remote("demo.counter", { version: 1, - input: Type.Object({ name: Type.String() }), + input: obj({ name: str() }), variants: { default: { - state: { count: Type.Number() }, - methods: { increment: method(Type.Object({ by: Type.Number() })) }, + state: { count: num() }, + methods: { increment: method(obj({ by: num() })) }, }, }, }); @@ -54,7 +54,7 @@ export const Counter = remote("demo.counter", { const host = createHost({ name: "server", self: { kind: "server" } }); host.serve(Counter, ({ name }, _principal, self) => { - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { variant: "default", count, @@ -82,7 +82,7 @@ Remote types can be reached from other hosts. Everything that crosses the wire h ```ts export const Chat = remote("pi.chat", { version: 1, - input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + input: obj({ session: str(), conversation: num() }), host: hostFamily("session:", "session"), // lives on `session:S1` for `{ session: "S1", … }` variants: { viewer: { @@ -91,7 +91,7 @@ export const Chat = remote("pi.chat", { participant: { state: { view: ConversationViewSchema }, methods: { - send: method(SendInputSchema, Type.Object({ submissionId: Type.Number() })), + send: method(SendInputSchema, obj({ submissionId: num() })), stop: method(), // no arguments, no result: `chat.stop(ctx)` }, }, @@ -119,10 +119,10 @@ Local types are reachable only inside their host. Their members may be anything: ```ts export const Index = local("pi.index", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }), members: { - search: method(Type.Object({ query: Type.String() }), Type.Array(HitSchema)), - onChange: method(Type.Function([ChangeSchema], Type.Void()), UnsubscribeSchema), + search: method(obj({ query: str() }), arr(HitSchema)), + onChange: method({ type: "function", parameters: [ChangeSchema], returnType: { type: "void" } }, UnsubscribeSchema), harness: opaque("pi.durable-harness@1"), }, }); @@ -181,7 +181,7 @@ A reload runs the factory again over the same object. Declarations that keep som ```ts host.serve(StatusStore, ({ session }, _host, self) => { const status = self.state("status", StatusSchema, { running: false, tasks: 0 }); - const errors = self.state("errors", Type.Array(Type.String()), []); + const errors = self.state("errors", arr(str()), []); const catalog = self.get(Catalog, {}); const db = self.resource("db", { version: 1, @@ -230,7 +230,7 @@ host.serve(Visits, (_, _p, self) => { // right host.serve(Visits, (_, _p, self) => { - const visits = self.state("visits", Type.Number(), 0); + const visits = self.state("visits", num(), 0); return { hit: async (ctx) => { visits.change(ctx, (n) => n + 1); @@ -240,7 +240,7 @@ host.serve(Visits, (_, _p, self) => { }); ``` -**Methods** are declared with `method(args?, result?)` in the contract. Without `args` a method takes none: callers write `ref.hit(ctx)` (or `ref.hit()`), and its implementation is `(ctx) => …`. Without `result` it returns nothing: the implementation returns nothing, and callers get `undefined`. `method(undefined, Type.Number())` takes no arguments and returns a number. +**Methods** are declared with `method(args?, result?)` in the contract. Without `args` a method takes none: callers write `ref.hit(ctx)` (or `ref.hit()`), and its implementation is `(ctx) => …`. Without `result` it returns nothing: the implementation returns nothing, and callers get `undefined`. `method(undefined, num())` takes no arguments and returns a number. ## Activation @@ -428,7 +428,7 @@ chat.release(); ## State ```ts -const todos = self.state("todos", Type.Array(TodoSchema), []); +const todos = self.state("todos", arr(TodoSchema), []); todos.change(ctx, (draft) => { draft.push({ id: "t1", text: "Write docs", done: false }); @@ -457,7 +457,7 @@ view.replace(ctx, rebuilt, { diff: true }); // always diff ```ts const db = self.resource("db", { version: 1, open: () => openSqlite(path), close: (db) => db.close() }); -const notes = self.state("notes", Type.Array(NoteSchema), { init: () => db.value.all("SELECT * FROM notes") }); +const notes = self.state("notes", arr(NoteSchema), { init: () => db.value.all("SELECT * FROM notes") }); ``` `init(ctx)` runs once the resources are open and the references live, before the object goes live, so no method, diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index 8e63d8d36..801a30c66 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -583,3 +583,392 @@ each; the head lookup and scans under it are at parity or faster in the micro-be memory, 55.5 µs persisted (two commits: the record and state created, then the reservation); a state change inside `run` 2.7 µs in memory, 21.3 µs persisted (one delta commit, as the gate's (c)), 3.2 µs deferred (stored with the outcome). + +## Slice 8: registries and `host.root` (2026-10-28) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `391e83fb1`, extracted at a path of the same length +(`/home/badlogic/pi-h8`); the build was measured in the worktree and, where the worktree itself skewed a result (below), +in an extract of the same sources at `/home/badlogic/pi-x8`. + +**Paths under every object** (`test/next/bench/calls.ts`, new: runs on HEAD as it is), median per operation over nine +bursts, heap allocated per operation, three alternating runs each, HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.80-2.81 µs, 6,894 B | 2.81-2.83 µs, 6,894 B | +| commit through a watched derive (`batch.compute`) | 10.34-10.43 µs, 15,312 B | 10.46-10.51 µs, 15,310 B | +| activation churn: get, live, release (an object with params) | 57.5-59.3 µs, 43,696-43,717 B | 58.1-58.9 µs, 43,782-43,787 B | +| retained per live object (10,000 kept) | 6,586 B | 6,602 B | + +The attempt keeps its registry reads, contributions, wake signals and writables in one record made by the first such +declaration, and iterates nothing when there is none: the first build allocated 1,020 B more per activation (six +arrays per attempt, an array iterator per `for … of` over an empty list, a copy from `splice`) and retained 72 B more +per live object; now +70 B allocated and +16 B retained (one field on the attempt, one on the object). + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, three runs) 9.4-9.8 → 9.5-9.9 µs median; its +pooled p95 16.5-17.1 → 18.4-20.8 µs comes from the run in which V8 tiers the commit path up (per run, after it: HEAD +10.7-11.3 µs, build 10.1-11.3 µs; the first measured runs 15.0-25.6 µs on either side, in another run each), the same +with HEAD's `batch.ts` and `derive.ts` in the build. The store gate (`store.ts`, three runs): every median within +0.4 µs (Chord store 13.0-13.9 → 13.0-14.2 µs, Chord host 17.3-18.2 → 17.3-17.8 µs, in-memory document 2.1 → 2.2 µs), +heap 3,498 → 3,144 KiB. Jobs (`jobs.ts`): start 8.4 → 8.5 µs in memory, 55.4 → 56.2 µs persisted; state change 2.7 → +2.7, 21.3 → 21.8, 3.4 → 3.5 µs. Links (`links.ts`, three runs): wire sizes, encodes, decodes and heap per message +unchanged; its round trips are one cold burst each and vary as much between HEAD runs (commit to frame, text codec, +0.055-0.076 ms). A warmed remote call (2,000 calls per burst, nine bursts, a memory link): binary 7.16-7.24 → 7.17-7.29 +µs, text 7.66-7.72 → 7.70-7.83 µs in the extracts; the worktree itself runs the same sources at 7.8-7.9 µs (binary), +an effect of the tree, not the code. + +**Registries and writables** (`test/next/bench/registries.ts`, new), median over nine bursts: `writable.set` delivered +to one watch 0.81-0.91 µs (2.9 KB allocated: the batch and the watch's call); a set of the same value 0.008 µs, 0 B +(one comparison: its batch work is a method of its own, so `set` allocates no closure context); one of 10 contributions +changed while the registry is watched 0.74-0.77 µs, of 100 1.52-1.59 µs (the table built again over every entry, one +compose per set: the counted `chord.registry.composes` is 1.00 per set). + +**Fix round 1** (2026-10-28), HEAD extract → build extract (`/home/badlogic/pi-x8`), CPU 6. `calls.ts`, two runs: local +method call 2.80 → 2.80-2.81 µs, 6,894 B both; commit through a watched derive 10.51-10.65 → 10.72 µs; activation churn +57.9-59.0 → 59.3-60.0 µs, 43,714 → 43,769-43,787 B. The derive diamond (five runs): median 9.4-9.8 → 9.5 µs, pooled p95 +16.4-16.7 → 19.0-19.2 µs. Per run, both have one slow run while V8 tiers the commit path up (HEAD its second, median/p95 +15.5/22.6-22.9 µs; the build its first, 16.4/26.7-27.2 µs) and equal runs after it (HEAD 8.8-9.6/9.6-11.2 µs, build +8.9-9.8/9.7-10.6 µs). `batch.compute` is not the cause: HEAD's `batch.ts` and `derive.ts` in the build give the same +p95, and swapping host files moves it without a pattern (code layout, not work on the path). `registries.ts`: +`writable.set` 0.87 µs, the same value 0.011-0.016 µs and 0 B (the seal check now comes first: a set of an ended +attempt's writable throws whatever its value); one of 10 contributions set 0.71-0.78 µs, of 100 1.43-1.61 µs (one run +in six 3.4 µs on either variant tried: JIT). + +## Slice 7a: the dependency model (2026-10-29) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `babe30a7e`, extracted at a path of the same length +(`/home/badlogic/pi-h7`); the build in the worktree (`/home/badlogic/pi-7a`). + +**Paths under every object** (`test/next/bench/calls.ts`), median per operation over nine bursts, heap allocated per +operation, three alternating runs each, HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.81-2.82 µs, 6,894 B | 2.80-2.82 µs, 6,894 B | +| commit through a watched derive | 10.43-10.70 µs, 15,310 B | 10.38-10.53 µs, 15,310 B | +| activation churn: get, live, release (an object with params and one dependency) | 58.5-59.9 µs, 43,716-43,719 B | 53.7-55.4 µs, 41,540-41,559 B | +| retained per live object with one dependency (10,000 kept, an ad-hoc probe) | 7,398-7,406 B | 6,931-6,936 B | + +An activation whose dependencies are live at the check does not park: one check of its bindings, then its own steps a +microtask later (as HEAD's wait did). The first build parked every attempt with a dependency (a waiter list, a +`parked_long` record, a microtask per wake even outside parking) and cost 62.0-63.6 µs and 580 B more per activation; +now a wake outside parking is a flag, the first-outcome promise is made only for whoever waits (the start barrier), +and an object keeps no `Set` of waits and no settlement promise per attempt (R18's), hence the lower numbers. + +**Links** (`links.ts`, three runs each): wire sizes, encodes, decodes, heap per message and retained heap per hold +(176.0 → 175.4-176.0 KiB) unchanged. Hold to live over two hops (mean of 100), binary: 1,000-item list 0.781-0.817 → +0.807-0.811 ms, 500 KB document 1.250-1.278 → 1.289-1.310 ms (+1-3%: the hold counted at its arrival, the index +check of the publication); text 1.545-1.580 → 1.521-1.573 and 3.299-3.379 → 3.317-3.393 ms. Commits to frames, +streamed appends, call round trips and reconnects within the runs' spread. + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, three runs) 9.4-10.1 → 9.5-9.6 µs median, +p95 19.3-19.9 → 19.3-19.4 µs. Registries (`registries.ts`, three runs): `writable.set` 0.82-0.97 → 0.77-0.87 µs; one of +10 contributions set 0.72-2.71 → 0.70-1.06 µs; of 100 1.50-3.46 → 1.44-3.40 µs (one slow run per side: JIT); +allocation within 20 B. A registry's last good composition keeps the list of contributions it was built from: the +list is replaced (not spliced) when a contributor comes or goes, so a composition costs no copy. + +**Fix round 1** (2026-10-29: revocation over paths, retry operations on the wire, the provider's own host-address +check for same-host remote requests, mapped values' provenance), `calls.ts`, three alternating runs, HEAD → build: +local call 2.81 → 2.79-2.82 µs (6,894 B both), watched derive commit 10.45-10.70 → 10.35-10.47 µs, activation churn +58.6-59.8 → 53.9-54.2 µs (43,716-43,720 → 41,559 B); retained per live object with one dependency 7,399-7,405 → +6,930-6,938 B. The address check runs only for a remote type whose consumer and provider `hostAddress` differ; a +retry's operation id is made per `ref.retry()`, and an object keeps its handled ids only once it was retried. + +## Slice 7b: reload in the kernel (2026-10-29) + +Node 24.4.1, one CPU (`taskset -c 6`), runs alternating. HEAD is `0c3675e64`, extracted at a path of the same length +(`/home/badlogic/pi-hb`); the build in the worktree (`/home/badlogic/pi-7b`). + +**Paths under every object** (`test/next/bench/calls.ts`), median per operation over nine bursts, heap allocated per +operation, alternating runs (the last two after the fixes below), HEAD → build: + +| Case | HEAD | build | +| --- | --- | --- | +| local method call (its ctx) | 2.80-2.83 µs, 6,894 B | 2.67-2.70 µs, 6,757 B | +| commit through a watched derive | 10.38-10.69 µs, 15,309 B | 10.22-10.39 µs, 15,312 B | +| activation churn: get, live, release (an object with params) | 53.6-55.3 µs, 41,539-41,560 B | 54.2-55.6 µs, 41,606-41,630 B | +| retained per live object with params and one dependency (10,000 kept, an ad-hoc probe, median of 5) | 8,696-8,700 B | 8,761 B | + +The first build cost 60.4-62.7 µs and 2,480 B more per activation: every attempt made an `AbortController` for its +routines and two maps (routine calls, job invocations) whether it had any. Now they are made when routines start or a +run is admitted, and the first dependency check's answer is a shared constant unless someone asks for it; what a reload +did to an attempt is one record made by the first such step. What remains: a few fields on the object (its params key) +and the attempt (its routines, its reload record, its first check), +46-70 B allocated and +61-65 B retained (0.7%). +A local call allocates 137 B less: the method path is split (`#invoke`) so the current version's call builds no +mapping closure. + +**Links** (`links.ts`, two runs each): wire sizes, encodes, decodes and heap per message unchanged. Hold to live over +two hops (mean of 100), binary: 1,000-item list 0.780-0.794 → 0.797-0.821 ms (+1-4%), 500 KB document 1.246-1.275 → +1.270-1.275 ms; commits to frames and streamed appends within the runs' spread. Retained per hold of 10: 175.8 → 176.2-176.5 +KiB (the hold keeps its request, to be placed again after a reload); the single-hold row 176.0 → 180.2 KiB is one +fixed amount (~4 KiB, code of the new paths), divided by ten in the other row. + +**Existing benchmarks**, HEAD → build: the derive diamond (`derives.ts`, two runs) 9.5-9.6 → 9.6 µs median, p95 +16.4-19.3 → 14.1-14.3 µs. Registries (`registries.ts`, two runs): `writable.set` 0.94-0.96 → 0.77-0.86 µs; one of 10 +contributions set 0.77-1.05 → 0.69-0.72 µs; of 100 1.57-1.58 → 1.54-3.30 µs (one slow run: JIT, as in 7a). Jobs +(`jobs.ts`, two runs): start 8.6-8.7 → 8.5 µs in memory, 55.3-56.1 → 55.3-55.6 µs persisted; state changes within 0.1 +µs. The store (`store.ts`): every median within 0.3 µs. The relay (`relay.ts`, three runs): tab B alone 0.014 ms both; +10,000 calls at once 530-723 → 491-785 ms (the spread of either side). + +**Reload** (`test/next/bench/reload.ts`, new): 20 plugins, 1,000 live objects (with params, a document, a derive and a +method each), one plugin reloaded 20 times; medians after two warm-up reloads, two runs: + +| Case | reload resolves | all live again | heap after 20 reloads, above after the first | +| --- | --- | --- | --- | +| swap, 50 of the 1,000 objects in the reloaded plugin | 1.58-1.59 ms | 1.64-1.65 ms | 332-338 KiB | +| swap, all 1,000 in it | 35.5-36.3 ms | 36.6-37.2 ms | 191-192 KiB | +| restart (a native each), 50 of 1,000 | 1.20-1.54 ms | 1.90-2.05 ms | 105-106 KiB | +| restart, all 1,000 | 20.0-20.2 ms | 35.8-40.8 ms | 57-58 KiB | + +A swap costs about 36 µs per object (its staging record, the new attempt's activation, the hand-over of its old one), +a restart about 20 µs until the reload resolves (staging and the start of each fate; the new attempts activate after +the drains). The heap after reloads levels off (swap, 50 objects: 332 KiB after 20, 437 KiB after 80, 550 KiB after +300 reloads): compiled code and feedback of the new facets, not objects (the GC tests collect old attempts, removed +objects, providers and facets). + +**Fix round 1**, CPU 6, alternating against HEAD. `calls.ts`, three runs: local call 2.80-2.82 → 2.66-2.68 µs +(6,894 → 6,757 B); watched derive commit 10.42-10.66 → 10.31-10.47 µs; activation churn 53.7-55.8 → 54.3-55.9 µs, +41,540-41,560 → 41,566-41,584 B (an `?? []` per end, found here, is gone: the round's first measurement allocated +41,698 B); retained per live object with one dependency 8,699-8,700 → 8,768 B. Links (a review measured binary +commit-to-frame +19% and small calls +6% on another CPU): `links.ts`, two runs, binary commit to frame +0.085/0.097-0.099/0.063-0.065 → 0.087/0.097/0.064 ms (append/change/removal), small call 0.038-0.039 → 0.035 ms. A +warmed ad-hoc probe (binary memory link, 2,000 calls and 1,000 commits per burst) measures no regression: call round +trip 9.8-10.2 → 9.9-10.1 µs, commit to frame 60.6-65.1 → 61.0-61.6 µs. Reload benchmark after the round: swap 50 / +1,000 objects 1.50 / 35.9 ms, restart 1.67 / 21.4 ms until the reload resolves (2.12 / 38.1 ms until all are live); +heap after 20 reloads 339 / 193 / 104 / 68 KiB. + +**Fix round 2**, alternating against HEAD. Stock `links.ts` on CPU 22, six rounds each before and after the fix +(medians, binary / text): before, small call 0.062 → 0.068 / 0.058 → 0.064 ms (+10%), commit to frame within 1% +(append 0.156 → 0.156, changed field 0.165 → 0.162, removal 0.120 → 0.119 ms binary). Cut-down copies located it: +calls alone, or after the snapshot, commit and stream sections, or after the hold-memory trials alone, measured +equal; only the full sequence showed the gap, and a CPU profile of its calls section put the difference in the +remote call path (`Hold.call` with the attempt's call inlined, about 6.7 µs per call). The cause was `Attempt.call` +delegating the invocation to a helper method shared with the older-version path; one body for every version, and +no `async` (a local call allocated about 250 B more as an async function), fixed it. After: small call 0.063 → +0.065 / 0.058 → 0.058 ms, append 0.156 → 0.157, changed field 0.164 → 0.163, removal 0.119 → 0.120 ms, hold to live +(1,000 items) 1.037 → 1.023 ms. `calls.ts` on CPU 6: local call 2.89-2.92 → 2.77-2.85 µs (6,894 → 6,660 B), watched +derive commit 10.44-10.58 → 10.65-10.81 µs (15,309 → 14,969 B), activation churn 54.8-55.1 → 54.6-54.7 µs (41,560 → +41,592 B); retained per live object with one dependency 8,700 → 8,777 B. Reload benchmark: swap 50 / 1,000 objects +1.84 / 33.7 ms, restart 1.31 / 22.3 ms until the reload resolves (2.18 / 41.0 ms all live); heap after 20 reloads +323 / 200 / 104 / 70 KiB. + +**Fix round 3: the commit-to-frame gap is V8's young generation, not frame work.** Astra (CPU 22, five pairs) measured +binary append commit-to-frame 132 → 161 µs, changed field 148 → 168, removal 112 → 121 on the stock `links.ts`. +- *Path length.* The HEAD copy `/home/badlogic/pi-hb` measured append 0.155 ms; Astra's `/tmp/s7b-hd-astra`, with + identical sources, 0.129 ms. A copy of the same tree at a path of Astra's length (17 characters) measured 0.130 ms, + at a longer one 0.155 ms: on the same code the result depends on the checkout path alone. Compare copies at paths of + equal length. +- *Stock numbers, equal paths* (`/tmp/s7b-hd-astra` vs a fresh build copy `/tmp/s7b-xx-astra`, CPU 22, three + alternating runs, means): append 0.128-0.129 → 0.155-0.156 ms binary, 0.106-0.107 → 0.116-0.117 text; changed field + 0.144-0.145 → 0.161-0.162 binary, 0.097-0.098 → 0.107-0.110 text; small call 0.061-0.063 → 0.065-0.066 binary, + 0.058-0.059 → 0.058-0.059 text. +- *Same work.* Heap allocated per commit is equal (191.9 vs 191.4 KB including the codec rows); so are the telemetry + records per commit (5.012, the same names: two validations, two link checks, one encode). +- *The difference is scavenges.* In the timed append loop HEAD ran 5 scavenges costing 3.6 ms in total, the build 3 + costing 21 ms (changed field: 2 for 2.7 ms vs 1 for 10.9 ms). At the loop's start the young generation was 33.5 MB + in HEAD and 67 MB in the build (134 MB by the changed-field loop): V8 grows it from the bytes surviving its + collections, and the build crossed that threshold a few scavenges earlier. Over the whole run the GC work is equal + (scavenge time 1,620 vs 1,605 ms, survived bytes 471 vs 480 MB, 285 vs 284 scavenges); where the expensive + scavenges fall relative to the timed loops differs. Tracing the GC (`--trace-gc-nvp`) moves HEAD into the build's + pattern by itself. +- *With the young generation pinned* the two are equal: means 0.139/0.139 ms (append), 0.145/0.144 (changed field), + 0.116/0.115 (removal), 0.065/0.064 (small call), HEAD/build; per-commit medians and p90s too (three runs, warmed + script): append 0.085-0.086 / 0.085-0.086 ms median, 0.101 / 0.101-0.103 p90; changed field 0.073 / 0.072-0.073, + 0.083-0.084 / 0.083; removal 0.060-0.061 / 0.060-0.061, 0.073 / 0.072-0.073 (binary; text alike, within 1 µs). +- *The extra live heap is code.* Build minus HEAD after a full collection: +249 KB right after loading the modules + (old space +229 KB, before any object), +290 KB after a snapshot section, +296 KB after 1,000 appends, +281 KB after + 1,000 edits, +345 KB after 70 hold trials (old space +259 KB, the rest bytecode and code as more paths compile). It + does not grow with commits or holds beyond the retained-per-hold and per-object figures above. + +So `links.ts` changed: each commit section runs 1,000 untimed commits first (`WARM_UP_COMMITS`; the removal section +starts with 1,000 more notes for them), and reports commit-to-frame as mean, median and p90 of the 1,000 timed commits. +The warm-up stays because the timed loop then runs nearer the young generation's steady size: the warmed medians +match the pinned ones (append 0.082 vs 0.085 ms, changed field 0.072 vs 0.073, removal 0.060 vs 0.060), and HEAD and +build agree in five of six sections, against two of six without it (run-to-run spread is about 1 µs either way). The +mean still carries the scavenges that fall in the loop. After the change, equal paths, CPU 22, three alternating runs, +HEAD → build (mean; median; p90, ms): + +| Section | binary | text | +| --- | --- | --- | +| append | 0.106-0.108 → 0.142-0.144; 0.081-0.083 → 0.090-0.091; 0.093-0.096 → 0.125-0.127 | 0.095-0.119 → 0.121-0.125; 0.078-0.080 → 0.080-0.082; 0.090 → 0.094-0.095 | +| changed field | 0.084-0.087 → 0.098-0.099; 0.071-0.075 → 0.072-0.073; 0.081-0.084 → 0.081-0.082 | 0.086-0.090 → 0.099-0.101; 0.067-0.071 → 0.070-0.071; 0.079-0.082 → 0.082-0.084 | +| removal | 0.103-0.104 → 0.107-0.109; 0.060-0.061 → 0.060-0.061; 0.071-0.073 → 0.072-0.073 | 0.100-0.102 → 0.077-0.103; 0.058-0.060 → 0.059; 0.067-0.069 → 0.067-0.070 | +| small call | 0.052-0.058 → 0.053-0.054 | 0.054-0.065 → 0.059-0.069 | + +The binary append differs unpinned (median +11%, p90 +31%) and is equal pinned: the young generation's size in that +section, not per-append work. To compare commit-to-frame across two copies, run both at paths of equal length, pinned +to one CPU, alternating, with the young generation pinned: + + taskset -c 22 node --min-semi-space-size=16 --max-semi-space-size=16 --conditions=source --expose-gc test/next/bench/links.ts + +Fix round 3 hot paths, `/tmp/s7b-hd-astra` → a build copy at a path of equal length, CPU 22, three alternating runs +(`calls.ts`): local call 3.69-3.71 → 3.56-3.63 µs (6,894 → 6,660 B), watched derive commit 17.39-17.50 → 17.41-17.53 +µs (15,310 → 14,967 B), activation churn 104.2-104.8 → 107.6-108.3 µs (+3%; 41,526 → 41,426 B); retained per live +object with one dependency 8,700 → 8,585 B (one table per version replaced the current version's separate member +map). Reload benchmark, same CPU: swap 50 / 1,000 objects 2.88 / 51.9 ms, restart 2.49 / 35.7 ms until the reload +resolves (3.22 / 65.7 ms all live); heap after 20 reloads 345 / 188 / 119 / 85 KiB. + +Sol's independent runs (CPU 24, five pairs of the stock `links.ts`) show the same commit-to-frame gap (binary append +135 → 162 µs, changed field 145 → 168, text 109 → 119 and 100 → 108): the young-generation effect above. Sol's +isolated round trips (`sol-isolated-links.ts`, nine bursts each, no telemetry) measured a binary call at 44.83 → 47.20 +µs (+5.3%) on the round-2 code. On this round's code, `/tmp/s7b-hd-astra` → a build copy at a path of equal length, +CPU 24, five alternating pairs, medians: binary call 47.23 → 45.74 µs (-3.2%; HEAD's samples are bimodal, 44.3 or +46.9-47.9), binary commit 92.68 → 93.28 (+0.6%), text call 39.02 → 39.76 (+1.9%), text commit 82.86 → 85.28 (+2.9%); +with the young generation pinned: binary call 45.45 → 43.85 (-3.5%), binary commit 92.73 → 93.06, text call 39.97 → +39.65, text commit 83.65 → 85.46 (+2.2%). + +**Rework: references are the only indirection** (decision of the same name). `/tmp/s7b-hd-astra` (HEAD) → a build +copy at a path of equal length, CPU 22, three alternating runs. `calls.ts`: local call 3.72-4.03 → 3.62-3.77 µs +(6,894 → 6,660 B); watched derive commit 17.46-17.92 → 17.53-18.40 µs (15,310 → 14,900 B); activation churn +104.7-107.9 → 114.7-118.3 µs (+9.6%, accepted by the owner, 2026-10-29; 41,507-41,526 → 41,473-41,501 B). The churn's object derives over its dependency's +member: that derive now attaches through its own reference's member (one `ReferenceSource` made, bound and ended per +cycle) where HEAD shared the provider's one member source; the allocation is HEAD's, the time is the new node's (a +copy of round 6, before the rework, measured 55.2 µs against HEAD's 53.5-54.6 µs and the rework's 58.4 µs on CPU 6). +Retained per live object with one dependency (an ad-hoc probe, 2,000 objects, median of five): 6,924 → 6,738 B; a +first cut made a source for every member at the binding and allocated four sets per source (8,627 B): sources are made +on first use or when a reference that had values detaches, and a source's watch, subscriber and pass-through sets are +made by their first user. `links.ts` (warmed, mean / median / p90 in ms), binary: append 0.107-0.113 / 0.082-0.083 / +0.095-0.097 → 0.141-0.142 / 0.089-0.090 / 0.123-0.124 (the young-generation effect above), changed field 0.084-0.086 / +0.071-0.072 / 0.081-0.082 → 0.098-0.099 / 0.072-0.073 / 0.081-0.082, removal 0.102-0.108 / 0.060 / 0.071-0.073 → +0.107-0.109 / 0.060-0.061 / 0.072; with the young generation pinned, append 0.140-0.143 / 0.086 / 0.100-0.102 → +0.143-0.147 / 0.085-0.087 / 0.101-0.103, changed field 0.091-0.092 / 0.072-0.073 / 0.082-0.084 → 0.091-0.092 / +0.073-0.074 / 0.082-0.084, removal 0.108-0.111 / 0.061-0.062 / 0.072-0.075 → 0.108-0.110 / 0.060-0.062 / 0.072-0.076; +text alike within 3 µs. Small call 0.058-0.060 → 0.053-0.056 ms. Retained per hold (both ends): 175.4-176.0 → +176.4-177.0 KiB with one hold, 175.8 → 176.4 KiB with ten. + +The 1 MiB call row (final review): one burst of 50 calls read 0.389 → 1.088 ms (binary, young generation pinned) in +one review run. Not per-call work: a probe with the row's setup and a collected heap before each of 20 bursts of 50 +(CPU 22, three alternating pairs) measured, per call without the collector's pauses, 0.27-0.31 → 0.30-0.31 ms binary +and 0.31-0.36 → 0.33-0.36 ms text, heap 23,025 → 22,707 B binary and 23,964 → 23,650 B text. A burst's time is about +0.3 ms per call plus 0.2-0.27 ms per call when a full collection falls in it (the arguments fill memory outside the +heap); one burst of 50 sees one or none, in either tree. `links.ts` now collects the heap before each of 20 bursts and +reports the median and p90; three alternating pairs of the new row, stock, binary median / p90 0.386-0.455 / +0.409-0.473 → 0.279-0.357 / 0.306-0.394, text 0.344-0.395 / 0.510-0.566 → 0.363-0.500 / 0.522-0.621; pinned, +binary 0.689-0.910 / 0.738-1.050 → 0.695-0.882 / 0.829-0.992, text 0.415-0.896 / 0.958-1.049 → 0.534-0.864 / +1.042-1.175. The row still moves by run with the collector's phase; compare it across several runs, not one. +The small call, measured apart (ten bursts of 1,000 after 3,000 warm-up calls, three alternating pairs): stock, binary +29.5-30.6 → 27.6-27.9 µs and text 25.1-25.9 → 27.5-28.3 µs; with the young generation pinned, binary 29.7-30.7 → +30.3-30.7 µs and text 23.6-23.7 → 23.9-24.1 µs. The stock text gap is the young generation's sizing, as for commit to +frame above (pinned, the two match); the snapshot before the final review's fixes measures the same. + +## Simplification audit (2026-10-30) + +HEAD `4b535d57e` → the audit build (A1, A2, A5, A9-A12), copies at paths of equal length, CPU 6, alternating. +`calls.ts`, three runs: local call 2.70-2.73 → 2.69-2.73 µs (6,660 B both), watched derive commit 10.41-10.75 → +10.42-10.74 µs (14,899 B both), activation churn 59.0-60.9 → 59.1-61.4 µs (41,473-41,482 → 41,478 B). Retained per live +object with one dependency (10,000 kept) 6,745-6,754 → 6,745-6,752 B; a first cut of the declaration-ordered +dependency list (A11) kept it on live attempts, +225 B, and is now let go after the dependency check. `links.ts` with +the young generation pinned: commit to frame, hold to live and retained per hold equal within run spread (176.4 → +176.2-176.6 KiB per hold). + +**Text-codec calls in isolation** (Sol's probe: a remote object over one memory link, 5,000 warm-up calls, ten bursts +of 1,000 small calls, then 200 warm-up and 30 bursts of 10 calls with a 1 MiB argument; binary first, then text; the +heap collected before each burst and the collections in it recorded: none in any burst), seven alternating pairs, +median of the pairs' medians: + +| Case | stock | young generation pinned | +| --- | --- | --- | +| binary small | 26.53 → 26.37 µs | 25.72 → 25.90 µs | +| text small | 19.31 → 20.95 µs (+8.5%) | 19.11 → 19.93 µs (+4.3%) | +| binary 1 MiB | 985 → 993 µs | 539 → 412 µs (bimodal: 187-607 µs per run) | +| text 1 MiB | 958 → 963 µs | 273 → 284 µs (221-352 µs per run) | + +Heap per call is the same (text small 15,171 → 15,160-15,202 B). The text small gap is not per-call work: the codec +alone (encode, decode and the message check of a call and a result message, 20,000 per burst) measures the same in both +trees (text call 1.55 µs and 52 B, result 1.52 µs and 171 B), deoptimizations and inlining decisions are the same, and +the gap follows the shape of the text decoder's `fromFields`, not what runs: the build with HEAD's `text.ts` matches HEAD +(19.0-19.2 µs), the one change in it that matters is the never-executed `T` (retry) case, and writing that case with an +object spread again (`...(f.length >= 2 ? { hops } : …)`) restores HEAD's 19.1 µs while other edits of the same size do +not. With text measured first the gap is 22.4-22.5 → 23.0-23.3 µs (+2-3%). V8 code generation, recorded as noise. + +## Slice 7c: plugins (2026-10-30) + +**The plugins component** (`test/next/bench/plugins.ts`, new): 20 plugins, one worker facet each (a store object over a +document, a derive, a method, a helper module and a copied contract; 80.9 KiB of scripts in all), Node 24.4.1. Medians +of seven runs; three runs pinned to CPU 8, one unpinned: + +| Measurement | CPU 8 | unpinned (32 CPUs) | +| --- | --- | --- | +| cold build of the 20 worker facets (one esbuild build each, all at once) | 240.1-240.7 ms | 26.4 ms | +| rebuild of one facet after an edit | 13.1-13.3 ms | 11.0 ms | +| evaluation of one facet (`vm.Script`, its source map registered) | 0.16-0.27 ms | 0.09 ms | +| reload of one plugin on a host (evaluation and `host.reload`, a pure swap) | 0.44-0.45 ms | 0.44 ms | +| heap after 100 reloads of one plugin, against after a warm-up | +230-235 KiB | +229 KiB | + +The heap does not grow per reload: 2,000, 4,000 and 8,000 reloads leave +897, +1,186 and +1,432 KiB (sublinear); the +same evaluations through plain `vm.Script` with V8's compilation cache off leave +420 KiB at 8,000 against +465 KiB +through `evaluateFacet` (V8's own per-script bookkeeping), and the kernel's reloads alone wander between +200 and ++600 KiB at any count. Old facets and their closures are collected (the GC test: 200 reloads, every facet but the last +two collected, the heap within 2 MiB of the warm-up). + +Fix round 1 (inline source maps without `sourcesContent`: positions only), CPU 8: the 20 scripts take 53.4 KiB (80.9 +KiB before); cold build 241.1 ms, rebuild 13.0 ms, evaluation 0.14 ms per facet, a reload 0.39 ms, heap after 100 +reloads +238 KiB. + +**Kernel hot paths** (per plugin at load and reload changes only staging), HEAD `18a155e2f` → build at paths of equal +length, CPU 6, three alternating runs. `calls.ts`: local call 2.69-2.71 → 2.71-2.75 µs (6,660 B both), watched +derive commit 10.34-10.36 → 10.43-10.49 µs (14,899 → 14,901 B), activation churn 58.93-59.58 → 59.44-59.73 µs +(41,452-41,470 → 41,470 B). `links.ts` binary commit to frame (median / p90): append 0.035-0.036 / 0.058-0.059 → +0.034-0.036 / 0.058-0.060 ms, changed field 0.027-0.028 / 0.031-0.033 → 0.027-0.028 / 0.031-0.033 ms, removal 0.022 / +0.026-0.028 → 0.022 / 0.026-0.027 ms; with the young generation pinned alike within 2 µs; text alike; small call +0.026-0.030 → 0.028-0.033 ms (pinned 0.030-0.035 → 0.030-0.032 ms); retained per hold equal (one binary hold 176.2 → +170.1 KiB). `reload.ts`: swap of 50 / 1,000 objects 1.44-1.73 / 30.83-32.71 → 1.52-1.53 / 30.55-33.25 ms, restart +1.01-1.05 / 15.54-16.87 → 0.95-0.99 / 15.90-16.02 ms until the reload resolves. + +## Slice 10: HTTP and workers (2026-10-31) + +`test/next/bench/http.ts` and `test/next/bench/workers.ts`, Node 24.4.1, CPU 6 (the forked workers inherit the pin). + +| Measurement | Result | +| --- | --- | +| keep-alive round trip, a JSON body: plain Node handler / a route (same process as the client) | build: 42.0-42.5 / 46.4-47.0 µs: +4.2-4.9 µs (9.9-11.7%); fix round 1: 42.5-42.9 / 62.8-64.4 µs: +20.3-21.5 µs | +| the same after 30,000 warm-up requests (fix round 1) | 30.9-31.3 / 43.4-44.0 µs: +12.1-12.9 µs; with an authenticator reading `request.headers`: +15.9 µs | +| the same, server and client in two processes (CPU 6 and 8; build) | 36.4-37.0 / 41.7-42.4 µs: +5 µs | +| 100 MiB file: `pipeline` of a read stream / `fileResponse` | build (64 KiB reads): 602-618 / 487-565 MiB/s; fix round 1 (256 KiB, pulled): 477-751 / 560-611 MiB/s | +| a worker's cold start with one shipped script: fork to `started` / to `ready` (median of 7) | 531-532 / 538-539 ms | +| the server's heap per idle worker (20 workers, one reference each) | 49.5 KiB | + +A request's overhead was 16 µs at first: `host.root` cost about 7 µs of it per request (a linked abort signal, a +`Promise.race` and an `awaitWithContext` per scope). Root scopes now take the caller's signal with one listener and +settle one promise (`root.ts`; behaviour and tests unchanged): 4.2-4.9 µs, about the bar of 10% of a round trip. The +rest is the request's scope (its abort controller, `CallContext`), admission and the response's header merge. The cold +start is mostly Node's startup stripping the types of the source tree it imports; the start timer (60 s) is far from +it. + +**Fix round 1** (the `fetch` core, the security baseline, Chord's listener as a native object). A route's overhead in +the standard run grew from +4.5 µs to +20-21 µs (+41 µs before the Node fast path: the listener reads Node's parsed +headers and makes a Web `Request` only when something reads `request.headers` or the body or a relay verifies; whole +bodies are written as strings, in one write with the head). Warmed, +12-13 µs; a CPU profile of the warmed route path +(per request, `test/next/bench/http.ts`'s route): the listener's request function 13.6 µs inclusive against the plain +handler's 8.9 µs, of which writing the answer 8.3 µs (the plain handler's `writeHead` and `end`: 9.1 µs); `serve` 4.2 +µs (the root scope 2.1, the URL parse 0.9, default headers 0.6, admission and routing 1.1, the answer 0.7); the root +scope's seal in a later microtask 0.9 µs. The baseline's extra headers cost 0.4 µs (a plain handler sending them). +The rest, about 5 µs, sits in no function of the component: Node's tick and microtask machinery (33 µs inclusive +against 18 µs for the plain handler). Opus's two trims are in (`lookup` decodes only segments with `%`; one abort +controller per request: the Node side's own is gone). Kernel rows, HEAD `bc9a20f84` → fix round 1, CPU 6, alternating: +binary 1 MiB call median 0.182-0.220 → 0.196 ms; reconnect of 1 / 100 references 1.02-1.10 / 79.1-80.3 → 1.04-1.05 / +81.3-81.5 ms; watched `writable.set` 0.836-0.868 → 0.861-0.863 µs; watched derive commit 10.42-10.58 → 10.36-10.60 µs; +activation churn 58.9-60.5 → 59.1-60.8 µs. + +**Kernel hot paths** (routes without a timer, `RouteHandle`, `Path` events, root scopes), HEAD `bc9a20f84` → build at +paths of equal length, CPU 6, alternating runs. `calls.ts`: local call 2.70-2.83 → 2.70-2.83 µs (6,660 B both), +watched derive commit 10.36-10.60 → 10.34-10.79 µs, activation churn 59.43-61.30 → 59.48-60.72 µs. `links.ts`: commit +to frame and encode/decode alike; reconnect of 1 / 100 references 0.94-1.15 / 80.8-84.0 → 0.94-1.15 / 80.6-86.5 ms; +1,000 parked calls 24.4-25.2 → 24.3-24.6 ms. `reload.ts`: swap of 50 / 1,000 objects 1.44-1.55 / 30.78-31.42 → +1.32-1.56 / 31.20-31.83 ms, restart 0.96-0.98 / 15.75-16.34 → 0.95-1.10 / 15.91-15.94 ms until the reload resolves. +`registries.ts` alike (one of 100 contributions set 1.47-2.45 → 1.48-3.41 µs, run to run). + +## Slice 12: the plugin model (2026-11-01) + +**The build's checks** (`test/next/bench/plugins.ts`, the 7c set: 20 plugins, one worker facet each, a copied contract +by relative path), Node 24.4.1, CPU 8, medians of seven runs, three alternating runs. HEAD `ebce9debd` (its bench +importing the contract by plugin name, which HEAD resolves) → this build: cold build of the 20 facets 257.6-257.8 → +257.2-258.8 ms, rebuild of one facet 13.2-13.5 → 13.3-13.9 ms. With the app's API set and the kinds named (every +import checked by version and kind, every resolved file by its kind segment; the plugins importing `@bench/app/v1`): +257.4-258.8 ms cold, 12.9-14.3 ms rebuild, against 258.2-259.7 ms without. The checks cost nothing measurable; no hot +path changed (the kernel gains two accessors read only by `testPlugin`). + +Fix round 1: the bench itself now builds with the app's API and the kinds (its plugins import `@bench/app/v1`; Opus +L5), and every package file's real path is checked against the plugin's `node_modules`. CPU 8, three runs: cold +build 261.1-270.2 ms, rebuild 13.6-14.4 ms (the first two runs as before, the third with a slower cold build on a busy +machine). diff --git a/packages/chord/docs/next/decisions.md b/packages/chord/docs/next/decisions.md index c437875fe..a9647718a 100644 --- a/packages/chord/docs/next/decisions.md +++ b/packages/chord/docs/next/decisions.md @@ -1530,7 +1530,7 @@ validation and copies the v2 kernel added around Delta. - **Slice 6a build reviews (2026-10-27, under the owner's delegation to the reviewers' majority):** two rounds of four build reviews (`/tmp/s6a-build-review{,2}-{astra,fable,opus,sol}.md`), merged into two fix lists (`/tmp/s6a-fix{1,2}-brief.md`, 28 and 16 items) by majority, a point raised by one reviewer taking that reviewer's - recommendation. Rules they settle beyond fixes of the spec: (1) **members across attempts:** a local object's state + recommendation. Rules they settle beyond fixes of the spec: (1) **members across attempts** (**superseded in part** by "References are the only indirection": the reference, not a member source on the object, follows the object across instances): a local object's state member is followed by its references and by other objects' derives through one member source per object and member name; when a new attempt goes live every member re-binds by name (one diff frame from the last value, consumers' derives recompute, a member the new variant lacks fails `invalid/member`), as a remote reference re-holds and gets a @@ -1763,12 +1763,717 @@ validation and copies the v2 kernel added around Delta. back: `host.serve(R, { compose, accepts: { [old]: { contribution?: (old) => current, value?: (current) => old } } })`; a reader bundling an older token gets the composed value mapped to its version (computed once per composition and version, only while such a reader exists); without a `value` mapping, or for a reader newer than the provider, its - value is failed `contract_mismatch` (reported). + value is failed `contract_mismatch` (reported); such a reader fails `broken`, not `transient` (nothing heals it before + a reload, which retries broken objects; build review 2026-10-27). - **Slice 8 S3: `self.registry` is a dependency, as `self.get` (owner, 2026-10-27: "isn't self.registry meant to wait like self.get?"; my spec had it as a plain read, a mistake against design §6, §7 and §17):** activation waits until - the registry has a value (served and composed once; an empty table composes too, at the serve), bounded by - `readyTimeout`; without one in time the object fails `transient` (`unavailable/undeclared`) and is retried when the - registry gets its first value (R18); once live, a later failure of the registry changes nothing (last good value). + the registry has a value (served and composed once; an empty table composes too, at the serve); after `load`'s + setups a registry without a value cannot get one before a reload, so a reader then fails at once `transient` + (`unavailable/undeclared`) and is retried when the registry gets its first value (R18), as `self.get` of a type not + served here fails at once (refined 2026-10-27 after the second spec review: no `readyTimeout` wait at start); once live, a later failure of the registry changes nothing (last good value). +- **Slice 7b's open points, as recommended (owner, 2026-10-29: "yes"; brief `slice-7b/`):** (1) the API is + `host.reload(facets, ctx)`, by plugin name (added, changed; a plugin missing from the list is removed), resolving with + a report of what each object did, rejecting with the code error and nothing changed; `load` stays the first call. (2) + A `writable` across a pure swap goes back to its initial value (the new factory makes a new one; no positional + matching). (3) The rollout across hosts is the app's: it calls `host.reload` on each host in order (server, workers, + clients); Chord guarantees per host (staging, commit, `installing`, rollback). (4) No install watchdog: `reload` + takes a context and the app passes its deadline. (5) The loader lives in Chord (`chord/next/loader`, slice 7c). (6) + Chord ships the app kit as optional subpaths: `loader` and `dev` (7c), `http` and `workers` (slice 10), `web` and + `tui` (slice 11); telemetry (slice 9) last. +- **Slice 7b's build decisions (owner, 2026-10-29, answering the builder; `slice-7b.md` §13):** a loaded plugin passed + as the same facet object (`===`) is unchanged and untouched, any other facet value for a loaded name is changed; + activating (not parked) and backoff-waiting objects of a reloaded plugin are retried; the report's `parked` covers + swaps and retries (their first dependency check), restarts are reported at once; a swapped-out attempt is sealed + once its calls settled (bounded by `closeTimeout`) and the swap ended, its job invocations finishing on their old + code; a stuck old routine call is reported `drain_timeout` and fenced by the old routine calls' shared token; a swap + to another variant sends holds a fresh `bound`; a native restart ends in-memory runs `aborted { reload }` while + persisted ones follow 6b R3 (stopped, resumed per `resume`, no restart counted); staging is synchronous + (`installing` tested from a staged factory); a removed object's member sources and pinned keys are handed to the next + object of its identity (fix round 3: one hand-over per identity for removal, version refusal and an object ending + while requests wait, `slice-7b.md` §13 item 11); keyed members' versions are built in 7b (L1's keyed edits and an accepts `keyed` section, + decision "Keyed state members, local contracts only"). The builder's own choice, accepted by the owner (2026-10-29): + adapted state members are derives per object and version, not per binding. (**Superseded in part** by "References + are the only indirection": the hand-over of member sources and the per-version sources that outlive an object go.) +- **References are the only indirection (owner, 2026-10-29: "go"; after seven fix rounds of 7b found one edge case after + another in sources handed between objects):** supersedes in part 6a's "members across attempts" (one member source + per object and member name) and slice 7b decisions 8 and 11 (member sources and pinned keys frozen and handed to the + next object of the identity; per-version outward sources owned by (identity, version); `host/handover.ts`). The + problem: a consumer's watch or derive over `ref.count` was attached to the referenced object's own document or derive + (the reference was bypassed), so every change of what the reference points at needed a stable node somewhere else + (`MemberSource` on the object, then sources that outlive the object and move to the next one). The rule: (1) a + consumer's watches and derives over a referenced object's member attach through the reference's member, which passes + the subscription straight through to the object's own document or derive (no extra hop on commits). (2) When what the + reference points at changes (a new object instance after a factory re-run: reload, retry, restart; a new object + after the type was removed and served again; another variant), the reference moves its subscriptions to the new one + and sends one frame, a diff from the last value. (3) While it points at nothing, its members keep their last values + (a pointer, no copy): while loading (a restart, a retry, a swap whose new instance is not live yet) its watches get + no call; while failed (a live failure, its version refused, its type removed, a failed dependency) one `failed` call + with the cause's error, and they stay open (replaces 6a's "no frames while failed"); a reference that never had a + value reads `not_ready` while loading and its failure's error while failed, and its watches wait for its first values + (a member the new variant lacks fails `invalid/member`, as before). Keyed watches and pins re-pin on each new instance the same way (the + cross-attempt key sources of 6a go; a key keeps its last value until the new instance loaded it; a dependent's + factory pin is still loaded before the dependent goes live). Once hydrated, a reference has last values for every state member of its bound variant, read or not: when it + detaches it takes them at its version from the outgoing instance (through that instance's mapping for an older + version), keeping no node of it. After a `failed` call, the next good value arrives as `[["r", value]]` even when + unchanged; a watch opened while failed starts at the last value and gets the failure once. No source, mapping, + document handle or pin of an instance outlives it through a reference: a detached reference keeps only plain last + values and its failure; releasing it drops them and its bindings; stopping a keyed watch drops its pin. (4) The same rule over a link: a replica keeps its last values + readable while its hold is failed. (5) An older consumer (`accepts`) attaches to the object instance's shared mapping + for its version: per member one derive `(current) => old`, made on first use, dropped when no reference of that + version is left, gone with the instance (one map and one diff per commit per version, whatever the number of old + references); a version the instance does not accept gives no member (unbound, last values kept). A reference knows + its version from the contract token it was made with. Nothing outlives its object instance; nothing is handed over. + Kept: requests never end on a code change, healing on rebind, `accepts` for arguments and results. A simplification + audit of slices 6a to 7b follows 7b's commit (owner: "how much other such bullshit was added in the last slices"). + Refined after the gap checks (Astra, Sol, Opus) and the build (owner, 2026-10-29: "i accept the recommendations"): + a detaching reference follows the old instance until it stops changing (its seal after admitted commits, a restart + after its drain), then takes its last values for every member of its bound variant, read or not, never from an + instance it never followed; after a `failed` call the next good value is `[["r", value]]`; a watch on a reference that + never hydrated resolves at its first binding (rejects on release, revocation, close, abort); release drops last + values and bindings; registry values and job state are not affected. **Derives over a failed reference member compute + on its last value** (G2 (A), as 6a): a contribution built on it stays in its table; only the reference member's own + watches get the `failed` call, and `ref.state` shows the failure. **Activation churn +9.6% accepted** (CPU 22, 104.7-107.9 + → 114.7-118.3 µs, allocation equal): an object whose derive reads another object's member makes one pass-through + node per reference member per instance; calls, commits, links and memory are within the bar. Two references to one + key hold two pins (counted on the key's one source; the document is loaded and kept once). **Binary append commit-to-frame + median +10% accepted** (owner, 2026-10-29; CPU 10, 0.031 → 0.034-0.035 ms): equal with the young generation pinned, fewer + scavenges over the whole benchmark (V8's young-generation sizing, slice 7b fix round 3). +- **Per plugin at startup and reload (owner, 2026-10-30: "yes", "1. lgtm 2. yes"; analyses + `/tmp/per-plugin-{opus,fable}.md`):** supersedes "Rollback is all-or-nothing per host" (2026-10-2x, 7b §2, design §21) + and 7c's Q2 ("a facet throwing while evaluated fails its host's whole step"). Why all or nothing was chosen: per plugin + was decided first and p2's plugin reviewers found counter-traces (a dropped argument field written into a transcript; a + copied Durable document written by two versions; a provider's failed factory leaving the harness hanging; mixed copied + code and registries); v2 has since closed them (closed argument schemas for every caller, Q12, Q15; version binding + with waiting, 7a; ownership of documents; registry versions with `accepts`, Q8; copied cross-plugin files limited to + contracts without dependencies, 7c). Rules: (1) **at startup** a plugin whose build, evaluation or setup fails is left + out; `load` resolves with a report `{ failed: [{ plugin, error }] }` and never closes the host for it (the core facet + included: base code decides whether that is fatal); its consumers wait `failed { waiting }` as for any missing plugin. + (2) **At reload** a plugin whose new code fails (build, evaluation, setup, a factory in staging, a declaration error, + `checkAccepts`) keeps its old code (its loaded facet, unchanged), and every other plugin of the reload applies; the + report lists it (`failed: [{ plugin, error }]`, as `load`'s; an added plugin that fails is left out); staging retries without it. (3) **The closure rule** (Opus): after + staging, a changed plugin whose recorded requests would not bind because the plugin they need was kept old or left + out is kept old too, so a consumer edited together with a broken provider keeps working on its old code. Its "missing" + branch (owner, 2026-10-30: "keep", against narrowing it to names served before): a request nobody serves keeps its + plugin old whenever any plugin of the reload failed, since a failed new setup may be what would have served it. (4) What no + rule detects: a consumer relying on new behaviour of a kept provider whose contract (version, fingerprint) did not + change: the same risk as a rollout across hosts, documented. (5) `rollout` keeps a plugin dirty while it was kept old + on any host. Built with slice 7c (the kernel's `load` and `reload` change with it). +- **Slice 7c's points (owner, 2026-10-30; `slice-7c.md` §5):** (1) esbuild is an optional peer dependency of Chord, + imported only by the plugins component. (2) **No dev/prod split** (owner: "chord apps are supposed to not have a + prod/dev split. they are supposed to be self modifying applications"): watching, rebuilding and rolling out are core + machinery; one component `chord/next/plugins` (discovery, build, evaluation, `pluginFile`, the watcher, `rollout`) in + `src/next-node/plugins/`, exported with v2's entry point; it turns V8's compilation cache off when created; reloads + stay explicit (the watcher marks, never applies). Supersedes the separate `loader` and `dev` subpaths of "Slice 7b's + open points" (5), (6). (3) `installDependencies(plugin, ctx)` as coding-agent installs package dependencies (`npm + install --omit=dev --legacy-peer-deps --ignore-scripts --no-audit --no-fund` in the plugin's real directory, the + command overridable). (4) Cross-plugin imports: any file of a discovered plugin, copied, documented as contracts and + side-effect-free modules only, listed by the build, no static check yet. (5) `pluginFile` answered per facet by the + `require` evaluation passes. +- **Slice 7c build: an abort after the commit names what it applied (owner, 2026-10-30: "yes"; fix round 1, item + 11, Astra):** `host.reload` still rejects `aborted` when its context aborts after the commit (the commit stands, only + the wait ends), and the error carries `details.committed`, a `CommittedReload { added, changed, removed, failed }` + (plugin names; `failed`: kept on their old code). Why: the plugins component keeps per host the facet each plugin + runs and adopts it when the reload resolves; on this rejection it would keep the old facet and pass it at the next + reload, swapping the committed plugin back silently. Rejected: resolving with the report after the commit despite + the abort (it would change the rule that an abort after the commit rejects). Examples for 7c (owner): full coverage, + each example a folder with the host code and real plugin directories (35-39). +- **Slice 10's points (owner, 2026-10-30; `slice-10.md`):** (D1, "a"; `httpServer` **superseded** by `createHttp`, decisions "The HTTP core is a fetch handler" and "Settings are arguments") the HTTP component defines the registry `Http` + (`"chord.http"`) and serves it through a facet of its own (`http.facet`, plugin name `chord.http`) that the app loads + with its plugins; its `compose` builds the router; the server reads it with `host.registry(Http, ctx)` per request. + (D2, "a") Wraps are built in slice 10 as designed (decision "HTTP routes and wraps", design §23): a contribution's + `wraps` run around routes in the component's phases (`observe`, `headers`, `transform`, outermost first), within a + phase in contributor order, by default around their own plugin's routes (`scope: "own"`; `"all"` explicit; `route` + narrows to one, a missing one a reported conflict). The server's `headers` option stays for every response, also + those no route makes (a 404, a refused request). + (D3, "a"; **superseded** by "The HTTP server is a native object": the `Authenticators` registry) Authentication is one app function, `authenticate(request, ctx) => P | undefined`, called once per request + and per upgrade; no Chord registry of resolvers (p3 composes its own from sharing, devices and the relay, with its + own precedence rules). + (D4, "k": a) Request bodies: `request.json(limit?)` (parsed JSON, 1 MiB by default, malformed: 400) and + `request.body(limit?)` (an async iterable of byte chunks, for uploads); nothing is read until a route asks; past its + limit the read stops and the request is answered 413; a disconnect aborts `ctx` and ends the iteration. + (D5, "a") Responses are descriptions: `json`, `html` (a full document; replaces the design's `page { title, body }`), + `bytes` (`mime`, `size`, `read(offset, length, ctx)`, `download?`, `ranges?`; 206/416, `HEAD`, safe + `Content-Disposition`, 64 KiB reads with backpressure, stopped on disconnect), `redirect`, `error`; `headers` on every + kind (cookies, a preview's own CSP); `fileResponse(path, options)` over a file; a throwing route is a 500 without its + message. + (D6, "a") Plugin WebSockets are `sockets` in the `Http` contribution: `{ "/terminal": (socket, request, ctx) => … }`; + the component checks the origin and runs `authenticate` (no principal: refused, 403, before plugin code), upgrades + and hands the plugin a `ws` WebSocket for its own protocol; patterns as routes; wraps do not apply; the links path is + the component's (`host.accept`). The design's channel component (acknowledged byte channels) waits for a second user + and would be built on top of it. + (D7, "a") `ws` is an optional peer dependency of Chord (pinned exactly), imported only by `next-node/http`, as + esbuild for plugins. + (D8, "a") Route patterns: exact segments, `:name` (one non-empty segment, percent-decoded, `params.name`) and a + trailing `/*` (the rest, possibly empty, `params["*"]`); the most specific match wins, segment by segment from the + left (exact before `:name` before `*`); one pattern from two contributions is a reported conflict, the first by + plugin name wins; `HEAD` falls back to the `GET` route. + (D9, "a") Worker keep-alive: the child reports `host.activity` and the app's `busy` counts (a JSON watchable of + numbers, e.g. Durable's active tasks) whenever they change; the server stops a worker whose counts were all zero for + `idleMs` (default 5 min, checked every second), never while it starts; a stop sends `shutdown` (the host closes), + the process tree killed after 5 s. + (D10, "a"; **superseded** by R2 below: no automatic restarts, no route backoff) A failed worker start uses the kernel's route backoff (1 s doubling to 60 s while holds exist, + `resolve_failed` reported); no sticky failure; `workers.state` shows the error and when it is tried again. + (D10b, "b") `host.route(Kind, options, ctx)` returns a `RouteHandle` with `retry(address)`: it ends that address's + backoff, so `resolve` runs now if the address has holders (a Retry button for a worker that failed to start, wired + by the app to the handle; the workers component has no `retry` of its own). + (D11, "a") Workers never build: the server's plugins component discovers and builds every plugin; a worker's start + message carries its kind's current scripts (`Built`, plain data) and a reload ships new ones (D12); the child's + `plugins` only evaluates (no esbuild, no watcher). So plugins are defined where the server can read them, one set for + every host (owner: "no cwd specific plugins, which might be alright; that way workers can live anywhere"); project + plugins (design §26 #14) would be discovered and built by the server too. A worker on another machine needs the + plugin directories there only for `chord.external` packages and `pluginFile`. + (D12, "a") A rollout step returns one report per host, `{ hosts: [{ host, report }] }` (7c's `RolloutStep` changes; + the in-process server step returns a list of one); the workers step reloads every running worker in parallel; a + worker that closed during it is listed with its error and does not stop the rollout; a plugin's dirty mark clears + only when it applied on every host of every step. + Spec review (Fable, Opus): (R1, "a") `RouteHandle.suspend(address)`: while suspended the route does not resolve the + address, its holders stay `loading`; `retry(address)` ends the suspension; `workers.stop` suspends, then stops (an + archived session stays stopped while a tab still holds it). + (R2, owner: "take it"; **supersedes D10 and the kernel's route backoff**) **Workers start on demand and never + restart by themselves** (owner: "if it crashes, chances are it had a good reason to crash, and restarting it won't + fix that"; D10 as recommended was a restart loop whenever a consumer holds). A worker starts for a hold of an address + that has not failed (the first, or one after an idle stop or `stop`). A crash or a failed start marks the address + failed with its error until `workers.retry(address)` (`RouteHandle.retry`) or a rollout that reaches workers (new + code may fix it, as a reload retries failed objects); its holders and new holds see `failed` with that error, nothing + forks. Kernel: a route has no backoff timer; it resolves only for a hold, on `retry`, or after its link dropped while + held, and a rejected `resolve` fails the route's holders with its error (no endless `loading`); a network route that + wants to reconnect keeps that policy in its own `resolve`. The workers component owns its route + (`createWorkers(host, Kind, options)` calls `host.route` and keeps the handle) and answers a resolve of a failed + address with the stored failure at once. A start finding the lock still held by a draining old worker waits for it in + the child (bounded by the start timeout). R1's `suspend` and D10b's `retry` stay. + Other timed retries (owner, 2026-10-30: "keep them as is for now"): objects' transient failures (1 s to 60 s while + held), `self.on` handlers that throw (1 s to 60 s while live) and `self.onLive` restarts (the same, reset after a + minute's run) stay; they heal real transient failures (a network down for a while) and cost at most one try a minute + each; revisit if one shows up as a loop in practice. + (R3, "a"; refines D9) The child reports only the flip: `{ idle: true }` when every count (`host.activity` and `busy`) + becomes zero, `{ idle: false }` when one becomes nonzero; the server's idle clock starts at `true` (a status message + costs about 6 µs on each side, a local call 2.9 µs: no report per call). + (R4; **superseded in part** by "allowed hosts only through contributions" and "The HTTP component's security baseline": no `allowedHosts` option, provenance by the socket; "a if configurable at runtime so allowed domains can be added and removed by a plugin") Before `authenticate`, the + component refuses (403) a `Host` that is neither loopback nor allowed, and an upgrade (`/chord`, a plugin socket) + whose `Origin` is not the request's own origin; `request.origin` comes only from an accepted `Host` (`http://` on + loopback, `https://` for an allowed host); `headers(request, principal)` gets the principal. Allowed hosts are the + server option `allowedHosts` plus every `hosts` list in `Http` contributions: a plugin adds and removes its domains + at runtime by contributing a writable (`self.writable`, design §17), the router recomposed on each change (a tunnel or + relay plugin names its public host). + **The app kit is malleable by plugins** (owner, 2026-10-30: "all of this should be malleable via plugins"; "yes"): + in `http`, `workers` (and slice 11's components) every option that is not bootstrap is either a contribution to + `Http` (routes, wraps, sockets, hosts; writables where it changes at runtime) or a function the component calls per + use (`authenticate`, `headers`, a worker's `start`, `idleMs`), which the app backs with its own registries; nothing is + copied out of a contribution or an option at startup. Bootstrap stays base code, changed by a restart: the listen + address and port, the links path, the worker entry module with its `execArgv` and `env`, the host kind workers serve. + (R5, "agree") `workers.state` per address key: `{ status: "starting" | "running" | "stopping", pid, startedAt }` or + `{ status: "failed", error: { message, code?, reason? }, at, stderrTail? }`; no entry when stopped or never started + (a clean stop removes it); `failed` stays until Retry or a rollout; plain JSON (a core object may serve it). + (R6, "b, common enough") `bytes` may omit `size`: chunked transfer encoding, no ranges (no `Accept-Ranges`), `read` + called with growing offsets until it returns an empty chunk (a zip built on the fly, a growing log, an export paged + from a worker). + Spec review round 2 (Fable, Opus, Astra, Sol): (Q1, recommended) holders of a failed worker get `failed { waiting }` + with the stored error as cause and stay on the address's path: a later link of the address (Retry, a rollout) binds + them without a re-hold; a `get` waits its `readyTimeout` meanwhile. + (Q2, "a") A rollout that reaches workers clears the failures from before it, then calls `RouteHandle.retry` for the + cleared addresses that have holders (they start with the new code and are listed in the step); an unheld one starts + at its next hold; a worker that fails during this rollout stays failed (no loop: one start per held failed address per + rollout a person triggers). + (Q3, "a"; refines R1) A `stop`'s suspension lasts while the address has holders; they see `failed { waiting }` + ("suspended"); with the last holder the record goes, and a later hold starts the worker on demand (as after an idle + stop); whether an archived session may be opened is the app's (`authorize`, its views). Memory stays bounded. + (Q4, "a") The start timer (60 s) covers fork, lock and connect (`started`) only; after `started` a slow `load` is + objects showing `loading`; `ready` sets `state` to `running` and lets idle stops begin; a `load` that never ends + shows `starting` until `stop` (no kill, no retry). + (Q5, "a"; refines R3) A worker's idleness is `references`, `jobs`, `lingering` and the app's `busy`, not `calls`: a + remote call runs on a hold (counted), local calls come from routines (not counted, §14), jobs, Durable tasks (`busy`) + and handlers under a hold; no message per local call, no timer. The first report goes with `ready` (`ready { idle }`), + so a worker idle from its start is stopped too. + (Q6, owner: "if sockets are a contribution, why keep them when the contribution is removed?": design §17's rule, an + entry lives as long as its contributor and consumers keep nothing beyond a use) A plugin socket is closed (1012, + "service restart") when the contribution that handed it out leaves the table (a reload changing or removing it); a + `/chord` link or socket whose `Host` is no longer allowed closes (1008) when the allowed hosts change. +- **Slice 11's points (owner, 2026-10-30; `slice-11.md`):** the surfaces are generic plumbing; the app keeps + everything visible (owner: "this one needs to be generic, so apps can do whatever they want, css, tui components, + themes"). (P1, "a") `chord/next/web` ships serving the surface (the base bundle, the plugins' web scripts, + stylesheets and assets, the page through the app's template, all as `Http` contributions), the browser's client host + with its connection loop, and loading and applying the plugins' web facets; not HTML, CSS, components, themes, + navigation, slots or layout. + (P2, "script tag") A plugin's web script is served as a file that registers itself + (`chordFacet("", "", function (require, module, exports) { … })`) and is loaded with ` + diff --git a/packages/chord/docs/next/dependencies/model.md b/packages/chord/docs/next/dependencies/model.md new file mode 100644 index 000000000..d763cd78c --- /dev/null +++ b/packages/chord/docs/next/dependencies/model.md @@ -0,0 +1,330 @@ +# Dependency resolution in Chord v2: one model (decided, 2026-10-29) + +Status: decided (owner, 2026-10-29); **built for slice 7a** (audit A1, 2026-10-30: `installing` is cut, staging being synchronous; A5: retry dedupe by a visited set and a hop budget; A11: `parked_long` for any long park) (§1-4 and §6-11 without what installs and reloads add: +§1's `installing`, §4's install-end and reload wakes, §3.9's reload handover and §5's removal, versions and rollout +are slice 7b's; folded into `design.md` §3, §5, §6, §7, §17, §21's start barrier, §22; `slice-7a.md`). The draft +history below is kept as it was. Sources: the owner's direction +("park until resolved"; "no difference between local and remote resolution: a link going is like a plugin +unloading"), the reviews (`/tmp/dep-mechanism-*.md`, `/tmp/dep-graph-*.md`, `/tmp/barrier-*.md`, +`/tmp/model/review{1,2}-*.md`), the code at pi `358f7d46a`. **Owner** marks an open decision. + +Draft 7 (round 6): (A) also retries in lockstep without finding the cycle (Fable); the probe's recovery after a +suppressed search becomes a choice between (B1) a re-search backoff for probes only (recommended) and (B2) a full +search lifecycle without timers, since `dead` notices alone need recovery state that eviction may not drop (Astra, Sol). +Causes are compared by value on wakes (Opus). +Round 7: a returned search is accepted while its initiator is parked, whatever its round (Astra: re-searches must not +invalidate searches under way); detection is eventual, not within one backoff period; the local walk stays (Opus). + +Draft 6 (round 5): cycles across hosts become an owner choice between (A) the owner's slice 4b decision (simultaneous +starts across hosts time out, no probe message), narrowed to one deadline, and (B) the probe, completed with edge +tokens, no revisits, and `dead` notices back to the initiator (Astra: a check that dies silently can lose the only +search that survived dedupe). (A) turned out to either poll chains across hosts again or miss a ring of `waiting` +attempts, so (B) is recommended, pending its review. + +Draft 5 (round 4, Sol's phantom cycle): the second round is replaced by a check along the recorded trail of +`(attempt, edge generation)` hops; a reported ring existed at one moment; probes grow with their path. + +Draft 4 after round 4 (probe only): round numbers per attempt, forwarded only when newer than the stored one; probes +honoured on parent and route links, dropped on child links; completeness stated under an eventually stable graph; the +repair caveat broadened to any concurrent edge removal; a new edge generation resets confirmation; a bounded display +trail. + +Draft 4 changes (round 3): the probe starts a fresh round on every new or changed wait edge (a ring closes at its last +edge, whose source finds it), keeps one seen entry per initiator, accepts only its current round, and states its +cost honestly; every host drops probes from untrusted links; local forwarding is iterative; no-placement remote +requests are re-placed when a host's first parent link connects; a cohort object that ends before its first outcome +settles the barrier; the last-good refinement is marked for the owner. + +Draft 3 changes (round 2): remote types without placement keep today's fallback (here if served, else the parent); +hydration stays before parking; no copied `retryAt`; a dependency `ended` breaks its dependent; a reload's handover +binds new requests before releasing old ones; retry passes only to `broken` and `transient` blockers, once per +operation; `installing` re-evaluated at every install end (commit or rejection); a registry's last good value dropped +when one of its contributors leaves; the probe with round ids, bounded seen sets, edge-generation triggers, attempt +ids on every non-live answer and relays dropping probes from untrusted links; one cycle mechanism, local and remote. + +## 0. The problem + +The same situation, "what I need is not there", has five outcomes today, depending only on how it is declared: + +| Declared as | Provider missing | Today | +| --- | --- | --- | +| `self.get(LocalType)` | no facet serves it | `self.get` throws, the factory throws: `broken` | +| `self.get(RemoteType)` | the worker does not serve it | the hold is refused `broken`; the dependent `transient`, polled 1-60 s forever | +| `self.registry(R)` | no provider | `transient`, woken at R's first value only | +| `self.registry(R)`, reader newer | | `broken` | +| a reload removes the type | | holders get `gone`: ended for good | + +And the startup barrier deadlocks an acyclic graph across two hosts (Sol, `/tmp/dep-graph-sol-barrier.log`): S's eager +A needs W's B, B needs S's C; W's hold for C waits for S's whole `load`, which waits for A. Only `readyTimeout` ends it. + +## 1. Requests and one resolver + +- **A request** is what a reference stands for: type (or registry), params, principal, the consumer's contract version. + Its holder makes it and owns it: an attempt (`self.get`, `self.registry`, keyed pins), a call (`ctx.get`), outside + code (`host.get`, `host.hold`, `host.registry`), a link's peer (`hold`). **It ends only when its holder ends it** (the + attempt ends, the call settles, `release()`, a getter gives up, the peer's link or origin closes, revocation, the + holding host closes), never because its provider is missing, went away or does not fit. +- **The destination by address** (round 1): a `local` type goes here; a `remote` type goes to the address its + `hostAddress` (computed from the consumer's own contract) gives: here if that is this host's address, whether served + here or not, else over the route for that kind or the parent (design §3). Every `remote` type declares + its `hostAddress` (owner, 2026-10-29: the optional `placement` of slice 3a is removed and the field renamed, and with it the fallback "here if + served, otherwise the parent" and its re-placement at the first parent link). A hold arriving for an address that is not this host's is relayed or + refused (`invalid/host_address`) as today; a relay never captures a hold because it serves the same name. +- **Then the selected provider is checked** (served, version or accepted version and fingerprint, `authorize`, the + provider's own host-address check). Version checks never look at a same-name registration elsewhere. +- **The publication gate:** a request made on this host, or arriving on its links, before the host's first + publication (§6) is not placed yet; it shows `loading` and is placed by the publication. The gate lasts only for the + synchronous setups, never for activation, so it cannot deadlock. +- **Outcomes of the check:** + + | The check finds | Request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address | unbound `not_served`: `failed { waiting }` | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere (code skew) | unbound `version`: `failed { waiting }` | + | the registry is served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` | + | the provider host is installing a revision and the hold does not fit its current code | unbound `installing` at the provider: `loading` | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params | `failed { broken, forbidden \| invalid }`, as today, never indexed | + + The unbound reasons that wait for a name (`unpublished`, `not_served`, `version`, `compose`) are indexed under it on + the host that decided them: the consumer's host for a local request; the provider's host for a hold that arrived + over a link (it already keeps such holds as refused: `remote/hold.ts`). `installing` is the provider host's own, + re-evaluated at its install's end; `no_link` lives on the path, where the next link holds it again. Indexes drop an + entry when its request ends; GC-tested. + +## 2. States + +`ref.state` keeps its four kinds; `FailureCause` gains one kind, `waiting`: + +```ts +type RefState = + | { kind: "loading" } + | { kind: "live" } + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } + | { kind: "ended"; error: ChordError }; + +type FailureCause = + | { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // the object's own code, a cycle, or authority: retry() or a reload + | { kind: "waiting"; error: ChordError }; // not the object's fault: no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event +``` + +| The request is | `ref.state` | Calls | +| --- | --- | --- | +| bound, its object activating or parked with no failed blocker | `loading` | park, bounded | +| unbound `unpublished` or `installing`, or its path has no link | `loading` | park, bounded | +| unbound `not_served`, `version` or `compose` | `failed { waiting }` | reject at once | +| bound, its object failed | `failed` with the object's cause, as today | reject at once | +| bound, its object live | `live` | run | +| ended by its holder, revoked, or its host closed | `ended` | reject | + +**Owner:** `waiting` as a new cause kind (Opus, Fable; recommended: the cause says "not your fault, an event will wake +you" and a UI can say what is missing) or no new kind: `transient` with no `retryAt` (Astra; today's wire already +carries a `transient` cause without `retryIn`, `remote-object.ts`), or `loading` with a diagnostic (Sol; parks every +call on a missing plugin for its full `readyTimeout`). + +## 3. Dependents: parked until their dependencies resolve + +A **dependency** is what an attempt declares and needs before it goes live: `self.get` (local or remote), +`self.registry`, keyed pins (`at(key)` on another object's keyed member). `self.follow` is the attempt's own step (its +attach runs Durable's code), not a dependency; a follower declares `retryOn` on the registry it reads the harness +through, or recovers by its own backoff. + +1. **Order, as today up to the check:** the factory runs; persisted singletons hydrate (the object's documents, kept + across attempts); the requests are made. The attempt is **parked** while any dependency is not `live` (hydrated): + + | Its dependencies | The dependent shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting, "waiting for X: " }`, `retryAt` undefined; the message keeps the nearest blocker and the root cause within the 500-character rule | + | one failed with `invalid/cycle` (a dependent off the ring) | `broken` with the same error: it would wait on a ring that never heals by itself | + | one failed with `forbidden` (an `authorize` refused it: authority, §3.6) | `broken` with the same error: only `retry()` or a reload helps (slice 7a fix round 1) | + | one `ended` (revoked, or its host closed) | `broken` with that error: the request cannot come back; `retry()` or a reload makes new requests (a guard: a dependency ends only with its holder's principal or host, so the dependent ends too; Fable) | + +2. **The dependency check is the activation boundary:** once every dependency is `live`, the attempt runs its own + steps (natives in order, follow attaches, member checks, derives), bounded by its own `readyTimeout` deadline, and + goes live. A dependency that leaves `live` after the check is treated as after live (R18): the attempt goes on. This is + R18's rule for live objects applied from the check on, not a rule of its own (owner, 2026-10-29: a dependency can + go away at any point of an object's life; the object stays live, its calls through that reference meet its state). So a parked attempt holds no + natives, and an attempt that opened natives is never parked. +3. **No timer while parked:** no dependency deadline, no backoff. The attempt's deadline is paused while parked (as + eager migrations pause it) and resumes with the budget left. +4. **Wakes, one rule:** the dependent re-evaluates on every state change of any of its dependencies (live, loading, + failed, a new cause compared by value, ended; Opus: otherwise a ring of waiting attempts passes its causes round and + round), updating what it shows; it leaves parking when all are live. The factory never re-runs + while parked. Wakes are coalesced per object and run after the batch that caused them; a change during the + attempt's own steps or its teardown is remembered. (As built in 7a, with the parent's note 5: a change while the + parked attempt checks or loads its pinned keys is remembered and checked again; after the dependency check, during + the own steps, a change is ignored, §3.2's R18 rule.) +5. **Kept while parked:** its documents (the object's), its declarations and requests, its seal token; no natives. + Ended with the object (its last holder; a host-lifetime object when its plugin is removed or the host closes). +6. **`broken` only for the object's own code** (a throwing factory, a derive throwing at activation, hydration, + `migrate`, a document fingerprint mismatch, an `open` throwing `broken(...)`), **cycles** (§7) and **authority** + (`forbidden`, an ended dependency). +7. **Retry passes on:** `ref.retry()` on a `failed { waiting }` object retries its `broken` and `transient` blockers, + hop by hop (the existing `retry` message over links), once per retry operation (an id dedupes diamonds); `waiting` + blockers pass it on; nothing else is touched. Retry of a `broken` or `transient` object starts a new attempt, as + today. +8. **`retryOn`** wakes failed objects as decided in slice 8 (a value call retries a `transient` or `broken` object); + while the object is parked `waiting`, a value call does nothing: its dependencies decide. +9. **Own-code reload of a parked object:** its plugin's reload replaces the parked attempt with the new factory's, + as one handover: the new requests are made and bound first, then the old ones released (round 2, Astra N4: an + unchanged dependency held only by this object must not end in between; today's `#activate` binds before + `#releasePrevious`). +10. **After live, R18 unchanged:** a dependency leaving `live` leaves its dependent live. + +## 4. Wakes: two events + +| Event | Re-evaluates | +| --- | --- | +| **a target changes state**: a dependency's object goes live, loading, fails, gets a new cause or ends; a hold's `bound`, `loading`, `failed` arrive, also after a re-hold on a new link | the requests bound to it, and their dependents (§3.4) | +| **a name becomes usable, or an install ends, on a host**: a publication (`load`'s setups, a reload's commit, an install's commit) serves a type, a registry, a version; a registry composes or maps a value at a version; an install ends without a commit (rejected) | the unbound requests indexed under those names; the provider's `installing` holds; a refused hold re-runs its start and sends `loading`, then `bound` or `failed` | + +No new message kind for wakes: `loading`, `bound`, `failed` and `retry` exist. The wire's `failed` carries the new cause +kind `waiting`. + +## 5. Removal, versions, rollout + +- **A reload that removes a type:** its objects end as today (natives close, reason `end`; callers rejected at once; + handlers drain). Their requests are not ended: each becomes unbound `not_served` and binds again if a later commit + serves the name. + - **Locally:** a `Binding` whose target ended by removal becomes unbound and accepts a second `bind`: member sources + re-bind by name (one frame from the last value, as on a restart), keyed pins re-pin, parked calls flush or + reject, presence re-lists. The one new piece of machinery. + - **Over a link:** a `failed` on a bound hold unbinds it (Fable: one message, one rule; the consumer applies it as + `#startOver` plus `failed`); the provider detaches the hold's sources and drops its unsent pending ops and queued + `bound`; the ack slots of `bound`/`state` messages already sent stay counted until their `ack` or the link's loss; + calls in flight on it are rejected and free their slots once. +- **A reload that stops accepting a version** unbinds only the requests on that version (`version`); the object stays + (its own plugin's reload still restarts it if it has natives, §21). +- **A type moved to another host kind:** the request carries the address its own consumer code computed; the old host + answers `not_served`; the consumer's own reload makes a new request. No redirect. +- **A consumer whose code moves to a new version** releases its old request and makes a new one (today). +- **Rollout, without revision fields:** each host knows only its own install. A provider host that installs keeps + holds that do not fit its current code `installing` (`loading`) and re-evaluates them when the install ends, by its + commit or by its rejection. A provider host that is queued but not installing yet answers a mismatch `failed + { waiting, version }` until its install commits and wakes it: a visible blip of at most one install, accepted (Opus). + An install never waits for activations (a reverse-order rollout would deadlock); a host whose own install was + rejected fails its own requests at once with the reason (design §21). The loader keeps its own duties (the desired + code set, digests, serialized installs). +- **Old calls across a reload** follow §21 unchanged: a pure swap lets them finish on old handles; a native restart + drains them first; a removal rejects their callers at once and drains the handlers. Calls are never resent. + +## 6. Startup: publication, then first outcomes + +1. **Publication:** `load` runs every facet's setup, then publishes all at once in one batch (types served, + registries served and composed once, contributions in their tables), then wakes (§4) after that batch. Nothing + activates inside it. +2. **No gate beyond it:** holds and `host.hold` wait only for the publication, never for the barrier. This ends the + startup deadlock: W's hold for C binds as soon as S's setups published; B goes live, then A. +3. **The barrier** (**Owner:** (a) or (b); recommended (a), three reviewers of four): + - **(a):** `load` resolves when every host-lifetime object of this `load` has a first outcome: + `firstOutcome(o) = o.live || o.ownFailure || (o.parked && o.blockers.every((b) => !b.local || firstOutcome(b)))`, + where `local` means executing on this host (an object here, a keyed pin still loading here), not the contract's + kind. So `CodeMode → Harness` waits for the harness's outcome; a wait on another host or on a failed dependency + ends the first pass. One `firstOutcome` signal per object, separate from the readiness dependents wait for (Sol), + counted once, never reopened by retries. An object of the cohort that ends before its first outcome (its plugin + removed, the host closing) settles its entry as ended; `host.close` settles the waiting `load` (Astra R3). + - **What (a) promises:** every host-lifetime object on this host had its first try, and the contributions of those + that went live are in their tables. **Not:** every tool that will arrive (a parked or slow contributor, tools found + in `onLive`, MCP; Sol's probe `/tmp/barrier-sol-first-outcome.log`). What a first turn needs is p3's explicit + admission check. + - **(b):** `load` resolves after the publication; Chord adds `host.startup`, a watchable of the host-lifetime + objects' first outcomes with their causes; the app decides (Fable). The same tracker, awaited by the app. + - p2 today resumes its harness without waiting for all host objects (`apps/p2/src/hosts/worker.ts`), because its + worker objects that need server types are reachable only after `control.ready()`. + +## 7. Cycles (decided 2026-10-29: no probe) + +On one host, cycles are found as today: at `self.get`, following the waiting edges of activating and parked attempts +(exact). Across hosts, a hold carries the activation chain it was made in, which finds every ring whose objects start +one after another. The remaining gap, a ring across hosts whose objects start at the same moment, is not detected: +without a dependency deadline it stays parked. It is made visible: an attempt parked longer than its `readyTimeout` on +a remote attempt that is itself only `loading` is reported once per parking episode as `HostError` `parked_long +{ object, waitingFor, since }`; nothing fails or retries. A cycle is a plugin bug; the guide gives plugin authors and +agents four rules: (1) a factory declares only what the object needs, never a type that needs it back; (2) +back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data flows up through state and +registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare worker types in their factory. + +Rejected (owner, 2026-10-29): the probe (edge-chasing searches with a check along the recorded trail; seven review +rounds, `/tmp/model/review*-*.md`; complete only with a re-search timer or a full search lifecycle, the most intricate +code in Chord, for a plugin bug) and the slice 4b timeout narrowed (it polls chains across hosts again, misses a ring of +`waiting` attempts, and retries in lockstep without finding the ring). + +## 8. Getters, holds, calls: one rule + +- **`host.hold`** returns a reference at once; only invalid params throw (also before `load`). A missing local type + gives `failed { waiting }` after the publication, `loading` before it. +- **`host.get`, `ctx.get` and `host.registry`** (owner, 2026-10-29, decision 2): resolve when live; wait while `loading`, + `failed { transient }` or `failed { waiting }`, up to their `readyTimeout` (default 60 s, the caller's choice), then + reject with the last cause; reject at once only on `failed { broken }`. On + success, rejection or timeout their temporary wait goes; `host.registry` returns the `RegistryValue` with its + ordinary watch ownership (no `release()`). A UI holds and shows `ref.state`; code that needs the object now gets it, with the + patience it passes. +- **Calls** on a `loading` reference park, each bounded by the reference's `readyTimeout` from when it was made; on a + `failed` one they reject at once with the cause; sent calls are never resent. + +## 9. Registries + +- The same resolver on one host: a reader's request for R at its version is bound while R is served and has a usable + value at that version (composed; mapped by `accepts[old].value` for an older reader), otherwise unbound + `not_served`, `version` or `compose`, or `loading` before the publication. Woken at every transition to usable + (`#valued`'s first-value latch goes). S2's `broken` and S3's backoff go. +- **Version skew within one host:** a reload of the registry's plugin alone leaves other plugins' contributors and + readers on their bundled versions; `accepts[old].contribution` and `.value` keep them working, unsupported + contributions are left out and reported, unsupported readers are `waiting` (`version`). +- A registry waits for a value, never for a particular contribution (an empty table composes). After live, a later + `compose` failure changes nothing for the reader: `.value` keeps the last good composition, **but only while every + contribution it was built from is still in the table** (round 2, Astra N2): when one of them leaves (its + contributor ended or was reloaded), the last good value is dropped and `.value` throws until `compose` succeeds, so + it never hands out a native whose contributor ended. `compose` represents a missing contributor as a value. +- Registries stay host-local (contributions are in-process values). + +## 10. Keep-alive, presence, revocation, jobs + +- A request over a link counts once in `host.activity.references` from its arrival to its end, bound or unbound; + presence (`host.references`) lists only bound references. +- Revocation ends the requests of a revoked principal, bound and unbound, at every hop; nothing revives them. +- `gone` stays only for revocation (a provider host closing closes its links and sends nothing; its consumers see a + drop, `loading`, and hold again on the next link). +- Persisted job runs of a parked host-lifetime object are loaded with its documents; `run` and `finish` start only + once it is live; the run count keeps the object as today. + +## 11. Timeouts after the model + +| Timeout or backoff | After | Why | +| --- | --- | --- | +| `readyTimeout` of an attempt | only its own steps; paused while parked | a slow dependency is bounded where it runs | +| `readyTimeout` of `host.get`, `ctx.get`, `host.registry` | kept | a caller gets an answer | +| `readyTimeout` of a parked call | kept | an action must not run long after it was asked for | +| identity backoff 1-60 s | only for the object's own `transient` failures | nothing else says the disk or network is back | +| dependency backoff, polling of refused holds | removed | recovery is an event | +| dependency deadline across links | removed | the simultaneous-cycle gap is a plugin bug, reported by `parked_long` | +| `parked_long` report | new: once per parking episode after `readyTimeout` on a remote attempt that is only `loading` | a diagnostic; nothing fails or retries | +| route backoff, heartbeat, `closeTimeout`, routine and `finish` backoffs, `linger` | kept | not dependency waits | +| rollout install timeout (not built) | a watchdog on the install only | never a deadline for dependencies | + +## 12. What changes in the design + +R18 (dependency failures `waiting`, no backoff, the deadline paused while parked, the dependency check as boundary, +wakes on every change, an ended dependency breaks), S2 and S3 (`waiting`, woken at every usable value), slice 8's +last-good rule (dropped when a contribution leaves), §3's destination table (by address; the publication gate), §6 (the +simultaneous-cycle gap: no probe, `parked_long`), §7 (`host.hold`, getters, calls), §21 (no `gone` on +removal; a dropped version unbinds only its requests; installs never await activations; rollout without revision +fields; the reload handover), the start barrier (R17, W6), `FailureCause` (+ `waiting`), `parked_long`. Unchanged: what an object's own code reload does (§21), R18 after live, +`retryOn` outside parking, `self.follow`'s R4, the call window and flow control, revocation, no client providers. + +## 13. Decisions (owner, 2026-10-29) + +1. The barrier: (a), `load` resolves at the first outcomes of this host's host-lifetime objects (§6.3). +2. Getters wait through `waiting` until their `readyTimeout`; calls on `failed` still reject (§8). +3. `waiting` is a new `FailureCause` kind (§2). +4. A pure swap whose new factory declares a dependency that is not there parks the object (its gate closes; holders + see `loading` or `failed { waiting }`; calls park or reject); no overlap of old and new code (§21). +5. The last good registry value is kept only while every contribution it was built from is still in the table (§9). +6. Cycles across hosts: no probe; `parked_long` and four authoring rules (§7). + +1, 3, 4 and 5 are the recommendations, taken when the owner said "just build this" (2, 6 decided in discussion). diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index 7429a66e9..e48701e53 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -52,7 +52,7 @@ versions and adapters · 28 building: slices · 29 telemetry · 30 cost model · | link | a connection between two hosts | | host kind / address | typed description of addressable hosts / one host of a kind (`Session({ id })`) | | facet | a plugin's declarations for one host kind: the types it serves, its contributions; no state or behaviour of its own | -| object type | a contract: name, params (which instance), host placement, members, version | +| object type | a contract: name, params (which instance), host address, members, version | | object | one instance per (type, params, principal key); `local`: per (type, params) | | principal | who calls; stamped by links, typed by the app | | reference | what `get` returns; keeps its object alive; live and hydrated when awaited | @@ -88,8 +88,12 @@ worker worker worker addressable by a typed address, started on demand | server | yes, one | its own declaration | runs | | worker | yes, many | stamped by the parent | started on demand by the app | -Clients serve only `local` types (they are not addressable): a host without a principal refuses to serve a remote type -(`load` throws; a remote object is owned by a principal of the host serving it; owner, 2026-10-24). A client's local +Clients serve only `local` types (they are not addressable): a host without a principal or without an address refuses +to serve a remote type (a declaration error: `load` leaves that plugin out and reports it, a reload keeps it on its +old code; a remote object is owned by a principal of the host serving it, owner, +2026-10-24; and placed by its host address, owner, 2026-10-29). No client providers for now (owner, +2026-10-28): client-side capabilities (a camera, WebMCP tools) go through an owner's requests that clients answer +(section 20, the Questions service). A client's local calls have no principal; what a client calls on the server is stamped with the client's authenticated principal. Clients reach workers only through the server, which relays; workers reach each other through the server too. A detached process (e.g. a terminal host) would be another child kind of the server. Things that are not hosts: a code @@ -97,7 +101,7 @@ mode sandbox (a worker thread inside a worker), remote environment daemons (thei the TUI sandbox behind its launcher (the launcher only passes frames). A Durable Object deployment is a mesh routed by the edge (section 24 and the research note). -### Host kinds, addresses and placement +### Host kinds and addresses ```ts export const Server = hostKind("server"); @@ -106,37 +110,66 @@ export const Session = hostKind("session", { id: str() }); // params like a export const Notes = remote("notes", { version: 1, params: { session: str() }, // flat record of required scalars - placement: ({ session }) => Session({ id: session }), // a typed function of the params + hostAddress: ({ session }) => Session({ id: session }), // a typed function of the params variants: { … }, }); -export const Catalog = remote("catalog", { placement: () => Server(), … }); +export const Catalog = remote("catalog", { hostAddress: () => Server(), … }); ``` - `hostKind(name, params?)` declares a kind of addressable host; its params follow the params policy of contracts (section 5). `Session({ id: "S1" })` is an **address**: `{ kind, params, key }`, the key built like an identity key (`session|S1`); two addresses are equal when their keys are. No host-name strings appear in app code. -- **`placement`** (owner, 2026-10-21) is a typed function of the params returning an address; `local` contracts have - none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). +- **`hostAddress`** (owner, 2026-10-21 as `placement`, renamed 2026-10-29) is a typed function of the params returning + the address of the host that serves the object, required on every `remote` contract (owner, 2026-10-29); `local` + contracts have none. It is not part of the contract fingerprint; a changed rule surfaces as a refusal (below). - **`params`** (owner, 2026-10-21; was `input`) select the instance among objects of its type and are part of its identity (section 5). - **The caller computes the address** and sends it in the `hold`; a host that relays routes by that address alone and - never needs the contract or parses params. **The provider computes it again** from the params and refuses an object - that is not its own: broken `invalid/placement` (misrouted holds are never forwarded). During a rollout the refusal - is handled like a mismatch (provider on another revision: park; same revision: fail; slice 7). -- **Where a reference made on a host, or a hold arriving on one of its links, goes** (first match; slice 4a): + never needs the contract or parses params. **The provider computes it again** from the params: a hold naming this + host whose params the provider's code places elsewhere (code skew) waits unbound `version` (`failed { waiting }`, + `invalid/host_address`, dependency model §1; slice 7a); a hold naming another host's address is relayed, never served + here (misrouted holds are never captured by a relay that serves the same name). +- **Every reference is a request** (dependency model, slice 7a): it has one holder (an attempt, a call, outside code, + a link's peer) and ends only when its holder ends it (release, a call settling, a getter giving up, the peer's link + or origin closing, revocation, the host closing), never because its provider is missing, went away or does not fit. +- **Where a reference made on a host, or a hold arriving on one of its links, goes**, by address alone (slice 4a; + slice 7a: no same-name check on the consumer's host): | Case | Goes | | --- | --- | - | the type is served here and its placement (computed here from the params) is this host's address, or the host has no address, or the type no placement | here | - | a reference made here (`host.get`/`hold`, `self.get`, `ctx.get`) to a served type placed on another host | on, as if not served here (next rows): worker S1 reaches `Notes { session: "S2" }` through the parent | - | an arriving hold for a served type whose params place it on another host | broken `invalid/placement`; the address the hold names only routes it | - | the hold names an address of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | + | a `local` type | here | + | a `remote` type whose host address (computed here from the params) is this host's address, served here or not | here | + | an arriving hold naming this host's address | here | + | the address (computed, or named by the hold) is of a kind this host has a route for | over that route's link (the address's params checked against the kind, then `authorize`) | | otherwise (a remote type) | over the parent link; a hold that arrived *on* the parent link is never sent back to it: `unavailable` | | no parent link: the host never had one | `unavailable` | | no parent link: it is down (a drop) | waits on the parent path, `loading`, until the next link (design audit, 2026-10-27) | - A `placement` or `authorize` that throws fails the hold broken with that error; an address whose params fail the - kind's check is broken `invalid/placement`. + A `hostAddress` or `authorize` that throws fails the request broken with that error; an address whose params fail + the kind's check is broken `invalid/host_address`. +- **The publication gate** (slice 7a): a request made on this host, or a hold arriving on its links, before the + host's first publication (`load`'s batch of declarations) is not placed yet: it is `loading`, and the publication + places it. The gate lasts only for the synchronous setups, never for activation, so it cannot deadlock. +- **Then the selected provider is checked** (here: served, the version and fingerprint, the params, the provider's + own host-address check; over a path: at the provider host). Version checks never look at a same-name registration + elsewhere. What the check finds: + + | The check finds | The request | + | --- | --- | + | a fitting provider | bound (to the object here, or to the hold over a path) | + | not published yet | unbound `unpublished`: `loading` | + | nobody serves the name at its address (or serves it as the other kind) | unbound `not_served`: `failed { waiting }` (`unavailable` "X is not served here") | + | no fitting version, a fingerprint that differs at the same version, a host address the provider computes elsewhere | unbound `version`: `failed { waiting }` (`contract_mismatch`, `invalid/host_address`) | + | a registry served, but its first `compose` (or the reader's version mapping) threw | unbound `compose`: `failed { waiting }` (§17) | + | the path has no link (a drop, a route resolving) | on the path: `loading` | + | `authorize` refused, invalid params, a cycle | `failed { broken }`, never indexed | + + The unbound reasons that wait for a name are indexed under it on the host that decided them: the consumer's host + for a request made there, the provider's host for a hold that arrived over a link (kept refused there; a `retry` + re-runs its start). A publication serving a name (`load`'s, or a reload's commit, which wakes only the requests of + the names it serves, slice 7b) wakes its requests, which run their check again + (a refused hold sends `loading`, then `bound` or `failed`); no new message kind. Index entries go when their request + ends (GC-tested). The consumer never re-holds a request by itself: no backoff, no polling. ### The low-level host API @@ -193,21 +226,25 @@ server.route(Session, { Do not pass `context.abortSignal` to `fork`: `host.close` aborts it, which kills the worker and makes Node emit an `error` event that crashes an app without a handler. Chord closes the link at `host.close`; the worker exits when its -parent link closes. `resolve` returns the link before the worker has loaded (the worker connects first, W6): a -`resolve` waiting for the worker's `load` while the worker's facets hold server types would deadlock. +parent link closes. `resolve` returns the link before the worker has loaded (the worker connects first, W6): made +later, the link would leave the worker's `hello` unread and the heartbeat would close it during a long `load` (slice +10's spec review; the deadlock once feared here does not happen: 7a's barrier ends its first pass on a wait on another +host). The workers component (§23) does this. | Call | Meaning | | --- | --- | | `createHost

({ address?, principal?, ...options })` | with an address: addressable, serves only objects placed there, acts as `principal` on its own host; without: a client. `P` is the app's principal type | -| `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, then waits for the start barrier (section 21); a throwing setup rejects and closes the host; runs once. Loaders are adapters; base code passes its core facet the same way | +| `load([{ name, facet }], context)` | facets with their plugin names (the plugin's `package.json` name, npm style; duplicates and invalid names are refused); runs every facet's setup, publishes all of them in one batch (types and registries served, registries composed once, contributions; requests waiting for the publication are placed after it), then waits for the start barrier (section 21: the first outcomes of its host-lifetime objects, slice 7a); a plugin whose setup throws is left out and reported (`load` resolves with `{ failed: [{ plugin, error }] }`, decision "Per plugin at startup and reload"); runs once. Loaders are adapters; base code passes its core facet the same way | +| `reload([{ name, facet }], context)` | the complete new plugin set after `load` (a facet passed as the same object is an unchanged plugin): staging, then one commit, then every object of a changed or removed plugin takes its fate; resolves with a report (a plugin whose new code fails keeps its old code, listed in `failed`); one at a time (section 21, slice 7b) | | `connect(link, { principal, trusted?, codec? }, context)` | the link to the parent, at most one open; once it closed (a drop), `connect` takes the next, and every remote reference over the parent is held again on it. `principal` = who the parent is | | `accept(link, { principal, trusted?, codec? }, context)` | a link from a child; holds and calls on it act as `principal` | -| `route(Kind, { authorize, resolve }, context)` | how to reach hosts of a kind (below); called before `load`, once per kind; work arriving on its links runs under `context` | -| `get(Type, params, context, { readyTimeout? })` | a reference once its object is live (section 7) | +| `route(Kind, { authorize, resolve }, context)` | how to reach hosts of a kind (below); called before `load`, once per kind; work arriving on its links runs under `context`; returns a `RouteHandle`: `retry(address)` resolves the address again (D10b), `suspend(address)` stops resolving it until `retry` (R1) | +| `get(Type, params, context, { readyTimeout? })` | a reference once its object is live, waiting through `loading`, `failed { transient }` and `failed { waiting }` up to its `readyTimeout` (section 7) | | `hold(Type, params, context, { readyTimeout? })` | a reference at once that follows its object until released (section 7) | | `close(context)` | closes the links (children, routes, then the parent), then ends every object at once and tears them down, the last to go live first (reason `close`); rejects with an `AggregateError` of the native closes that threw | | `revoke(r, context)` | revocation (section 4; slice 4b) | -| `root(fn, …)` | a call scope for code outside objects (a Durable hook, a CLI); built in slice 8 (**decision needed** on its signature, section 26) | +| `root(fn, context, { principal? })` | a call scope for code outside objects (a Durable hook, a CLI, an HTTP request): `fn` gets a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes (slice 8 D3, section 8); an omitted `principal` acts as the host, an explicit `undefined` runs anonymous (slice 10) | +| `registry(R, context, { readyTimeout? })` | registry `R`'s composed value for code outside objects once it is usable; waits for the publication, then while it has none up to its `readyTimeout`, then rejects with the last error (slice 8, 7a, section 17) | **Options of `createHost`:** `address`, `principal`, `principalKey` (section 4; the same function on every host), `telemetry` (section 29), `onError`, `closeTimeout`, `readyTimeout`, `heartbeat: { ping?, close? }`; later `revokedBy` @@ -216,7 +253,7 @@ and closed by the app with `store.close(ctx)` after `host.close`; the host never background, as a tagged `HostError`: `listener_error` (a watch listener threw), `delivery_failed` (a frame delivery threw), `activation_failed { object, failure }`, `close_failed { object, native, error }`, `close_timeout { object, native }`, `drain_timeout { object }`, `protocol_error { error }`, `resolve_failed { address, error }`; later routine -errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6) and `storage_failed { error }` +errors (slice 5), `derive_failed { object, member?, error }` (a derive's compute threw, once per failure episode, slice 6a), the registries' `registry_conflict { registry, message, plugins }`, `registry_failed { registry, error }`, `contribution_failed { registry, plugin, type?, error }` and `contribution_refused { registry, plugin, type?, version }` (each once per episode, slice 8), `object_failed { object, failure }` (a live object failing, slice 6a, `broken`; `transient` when a `self.follow` source failed, slice 6b R4; `failure` where the value is a `FailureCause`, `error` otherwise, 2026-10-27), `job_interrupted { object, job, id, restarts }` (a persisted run ended at the restart limit, slice 6b D6), `parked_long { object, waitingFor, state, since }` (an attempt parked longer than its `readyTimeout`, whatever its blocker, once per parking episode: a diagnostic of a ring across hosts started at the same moment or a long wait, slice 7a, audit A11, §6) and `storage_failed { error }` (slice 6a), which every app must handle: Chord never ends the process, the app exits with its own code for its supervisor or reopens. @@ -225,12 +262,12 @@ the app exits with its own code for its supervisor or reopens. | Timeout | Bounds | Host default | Override | | --- | --- | --- | --- | | `closeTimeout` | a native's `close` (then reported as a possible leak; the rest continues); the drain of an ending object's running handlers (callers are rejected at once) | 30 s | per native: `self.open(open, close, { closeTimeout })` | -| `readyTimeout` | an activation attempt (its wait for the previous instance's teardown included); a getter's whole wait; how long a call made while its reference is loading waits (the park deadline) | 60 s | per `get`/`hold`: `{ readyTimeout }` | +| `readyTimeout` | an activation attempt's own steps (its wait for the previous instance's teardown included; paused while the attempt is parked on its dependencies, slice 7a); a getter's whole wait; how long a call made while its reference is loading waits (the park deadline) | 60 s | per `get`/`hold`/`registry`: `{ readyTimeout }` | -The park deadline's per-type override is not built (only if a real case needs it). Timeouts of later slices -(routine handover 10 s, job `finish` retry backoff 1 s to 60 s, rollout install 60 s, provider-side `linger` per type) -belong to their mechanisms (**decision needed** on which stay host options, section 26). A route's `resolve` is -bounded by the app. +The park deadline's per-type override is not built (only if a real case needs it). Timeouts of later slices belong to +their mechanisms: the routine handover uses `closeTimeout`, a job `finish` backs off 1 s to 60 s, provider-side +`linger` is per type; a reload has no install timeout (it takes the app's context, slice 7b); a per-type drain +override only if a case needs it (section 26). A route's `resolve` is bounded by the app. - **Incarnations** are minted by Chord (uuidv7, code copied from pi-ai, no dependency), never passed in, and carried in the link handshake (`hello`): the other side sees a restart as a new incarnation of the same identity; @@ -246,7 +283,9 @@ bounded by the app. - **Worker keep-alive** works as in p2: the worker counts, the app's supervisor decides. **`host.activity`** (slice 4b; owner, 2026-10-24) is a JSON `Watchable` of `{ references, calls, jobs, lingering }`: holds arriving over links, method calls running on this host, running jobs, provider-side lingering objects. Counters are kept where the work - happens; a change marks it dirty and it is published at most once per microtask turn with changes. Core code + happens; a change marks it dirty and it is published at most once per microtask turn with changes. A hold placed + here counts from its arrival to its end, bound or not (one waiting for a missing provider keeps the worker alive; + slice 7a); a hold this host relays counts at its destination; `host.references` lists bound references only. Core code reports it plus Durable tasks (the core `Harness`); the server's supervisor stops the worker when all are zero for the idle time (p2: 5 minutes, checked every second). Chord provides the counts, no idle logic. @@ -267,12 +306,16 @@ host.route(Session, { aborted only when the host closes; a link it returns after that is closed at once. A `resolve` whose holds were all released meanwhile finishes and its link is kept (a worker start that produced a link finishes even if its first requester left). A `resolve` that never settles is the app's (it bounds its own `fork`); getters still time out. -- **Failure and re-resolve:** a rejected `resolve` (reported `resolve_failed`), a returned link already closed, or the - address's link closing while holds over it exist, starts the **per-address backoff** (1 s doubling to 60 s; a map - keyed by address, separate from objects' identity backoff). Meanwhile the holds are `loading`. Only a link that - completed its handshake resets the backoff. Without holds for the address nothing is resolved again, so passive - viewers recover without a user action while they hold. -- Chord never closes a route link itself except at `host.close`; stopping idle workers is the app's (`host.activity`). +- **Failure and re-resolve** (slice 10 R2, owner, 2026-10-30; replaces the per-address backoff): a route has no + timer. A rejected `resolve` (reported `resolve_failed`) or a returned link already closed fails the route's holders + with its error (`failed`, not an endless `loading`); the route resolves again only for a new hold, on + `RouteHandle.retry(address)`, or once after the address's link dropped while holds exist (a drop of a healthy link + recovers at once; whether a crashed worker comes back is the `resolve`'s answer). `RouteHandle.suspend(address)` + (R1) stops resolving an address (its holders stay `loading`) until `retry`. A network route that wants to reconnect + keeps its own policy in its `resolve`. The workers component answers a failed address's `resolve` with its stored + failure at once: workers never restart by themselves (§23). +- Chord closes a route link itself only at `host.close`, on the heartbeat and on a protocol error; stopping idle + workers is the app's (`host.activity`, the workers component, §23). - Work arriving on a route's links runs under the context given to `host.route(Kind, options, context)` (slice 4b). ### Links @@ -321,8 +364,8 @@ host.route(Session, { proxy buffers; acks kept it 0.0 s old); - a hold that binds waits for a slot; its snapshot and subscription are taken in one synchronous step when its `bound` is sent; bindings and waiting state take turns when both wait, so neither starves; - - a hold whose object leaves `live` while its `bound` waits must go back to pending; only a reload makes a live - object leave `live`, so slice 7 builds it (D1, owner, 2026-10-24). + - a hold whose object leaves `live` while its `bound` waits goes back to pending (D1, owner, 2026-10-24; built in + slice 7b with the reload, which makes a live object leave `live`); so does a bound one, unbound (§7, §22). Not built, by decision: a byte window, credit messages, snapshot admission or debt, an adapter `backlog()` API, `reset` from anyone. - **Calls: a call window N** (default 256, the same on every host) announced in `hello`: how many concurrent calls a @@ -340,7 +383,7 @@ host.route(Session, { ### Relays (slice 4a; decision "Relay flow control per origin", owner, 2026-10-24) -A host that receives a hold it cannot serve (the placement table above: a route or the parent) **relays** it: it holds +A host that receives a hold it cannot serve (the table above: a route or the parent) **relays** it: it holds it again on that link (upstream) with the arriving hold's principal and incarnation, and forwards between the two without parsing payloads. The provider keeps the window, the pending queues and the calls per origin; the relay never drops, folds or resets. @@ -348,8 +391,10 @@ drops, folds or resets. - **Origins:** the relay gives each downstream origin (a downstream link, and the origin its messages carry when that link is itself a relay's) a number, one counter per host, never reused; 0 is a host's own work. Every message upstream for that origin carries `origin`; the provider keys holds, calls, windows and acks by (link, origin, id), and - its answers carry the origin back. `origin` is honoured on every link (a worker may relay a child's hold to its - parent); principals and incarnations stay parent-only. + its answers carry the origin back. `origin` is read only on parent and route links, where a relay's messages + arrive (a worker may relay a child's hold to its parent); on a child link it is a protocol error (decision "Origins + only from relays", 2026-10-31); a per-origin record goes once it has no holds, calls or messages in flight; + principals and incarnations stay parent-only. - **Ids are not renumbered:** an upstream hold has the downstream hold's id under the origin's number; calls the same. Two tabs (two holds) of one principal are two upstream holds, each with its own encoders (R1). - **Nothing is parsed:** the relay routes by the hold's address and authorizes it; params, arguments, results, @@ -397,7 +442,8 @@ drops, folds or resets. - **JSON plus `Uint8Array` leaves**, with a bytes schema type, defined independently of the transport. Every link carries the same `Message` through its codec (section 22); parts are encoded at their origin; byte segments travel raw, never base64. Same-host calls (no link) pass values by reference. -- Schemas are TypeBox and must describe data (no callbacks, no native types, no runtime codecs). Method arguments are +- Schemas are plain JSON Schema built with `@earendil-works/pi-schema` (decision "The schema switch") and must + describe data (no callbacks, no native types, no runtime codecs); `format` is an annotation, never checked. Method arguments are validated at the provider for every caller against closed schemas (section 5, R7, Q12, Q15); results and state are never validated (section 30). Values are immutable by convention once handed over (Delta's rule): nobody mutates a value it got, and nothing is copied or frozen to enforce it. There is no development mode (R9): one behavior @@ -471,7 +517,7 @@ drops, folds or resets. ## 5. Object types ```ts -export const Notes = remote("notes", { version, placement, params, variants }); +export const Notes = remote("notes", { version, hostAddress, params, variants }); export const NoteStore = local("notes.store", { version, params, members }); ``` @@ -514,11 +560,12 @@ host.serve(Notes, ({ session }, principal, self) => - **Reserved member names:** `then`, `release`, `state`, `retry` and `variant` are refused at declaration (a reference has them; `then` would make an awaited reference thenable). - **Compatibility (strict, like documents; R6):** a hold carries only the consumer's contract `version` and - fingerprint, never a description. The provider binds when both equal its current ones, or (slice 7) when the + fingerprint, never a description. The provider binds when both equal its current ones, or (slice 7b) when the version is one it accepts and the fingerprint equals the one computed from its own history (section 27). Every - contract change, additive ones included, bumps `version`. Anything else is `contract_mismatch` (or `updating` during - a rollout, section 21). The fingerprint covers params, variants and members, not placement. A consumer whose contract - does not fit gets a mismatch on its own binding; the provider object and its other bindings are not affected. + contract change, additive ones included, bumps `version`. Anything else leaves the request unbound `version`: + `failed { waiting }` with `contract_mismatch` (slice 7a: not the consumer's fault; a publication that fits wakes it, + §3). The fingerprint covers params, variants and members, not the host address. A consumer whose contract does not + fit waits on its own request; the provider object and its other bindings are not affected. - **Variants:** a member name has the same kind (state or method) in every variant that has it, and a method the same signature (arguments and result) in every variant; state schemas may differ per variant (redaction). Enforced at the type level in `remote(...)` and at declaration at runtime. A remote factory returns one variant, `{ variant, @@ -526,7 +573,7 @@ host.serve(Notes, ({ session }, principal, self) => (`invalid/member` outside it). - **Params** (owner, 2026-10-21/22): a flat record of required fields, each `str()`, `int()`, `bool()` or `oneOf(...)` of string literals, constraints allowed (`str({ pattern })`, `int({ min: 0 })`); no defaults, optional, nested or - unknown fields. Checked for every caller with one compiled closed-object TypeBox check (a missing or extra field + unknown fields. Checked for every caller with one compiled closed-object pi-schema check (a missing or extra field fails `invalid/params`). The identity key is the field values in declared order, each escaped, joined with `|`. Params are never copied or frozen: the object keeps the caller's object, immutable by convention. - **Arguments (R7, Q12, Q15):** always validated with the provider's schema, whoever the caller is (trusted code is @@ -540,14 +587,14 @@ host.serve(Notes, ({ session }, principal, self) => - **Providers** (`bindProvider(Type, accepts)`, L1): runs `checkAccepts` once per load and reload and keeps the validated mappings; per call `checkArgs(provider, method, raw, boundVersion, context)` returns a tagged value, `{ kind: "method"; args; result? }` (current-shape arguments and, for an old-version caller, the result mapping back) - or `{ kind: "call"; args; call }` (the provider's own replacement of a removed method, invoked by Chord, slice 7). The + or `{ kind: "call"; args; call }` (the provider's own replacement of a removed method, invoked by Chord, slice 7b). The binding is held only by the served type's registration and by calls in flight (GC-tested; no global strong cache). - **Errors** (L1, slice 3a): `ChordError` with a code and, for some codes, a reason, a closed union that is part of the link protocol (adding one bumps the protocol version; a receiver rejects unknown reasons; a test pins the list): - `invalid` (`args`, `params`, `member`, `placement`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` + `invalid` (`args`, `params`, `member`, `host_address`, `reentrant`, `read_after_bulk`, `cycle`), `contract_mismatch` (`updating`), `forbidden` (`revoked`), `gone`, `not_ready`, `released` (`ended`, `closed`, `settled`, `version`, `discarded`; none for a holder's own release), `too_large` (`part`), `unavailable` (`locked`, - `park_timeout`, `busy`, `storage_failed`), `aborted`. A failed check names TypeBox's instance path ("unknown field", "missing", else TypeBox's - message; keys containing `/` or `~` make the path ambiguous, documented). Plugins throw `PluginError(code, message, + `park_timeout`, `busy`, `storage_failed`, `undeclared`: a registry no plugin serves here, slice 8), `aborted`. A failed check names pi-schema's JSON Pointer path ("unknown field", "missing", else + pi-schema's message; `~` and `/` in keys escaped as `~0`, `~1`). Plugins throw `PluginError(code, message, details?)` for codes of their own; anything else crosses a link as the plugin code `internal` with its message, never a stack or a cause; plugin errors never use Chord's codes. An error rebuilt from the wire is not copied and only its envelope is checked; a malformed envelope is a protocol error. @@ -562,7 +609,7 @@ host.serve(Notes, ({ session }, principal, self) => ### Lifecycle ``` -(get) → activating → live → ending → ended +(get) → activating (parked on its dependencies → own steps) → live → ending → ended └→ failed (backoff) → activating … live → failed (a runtime failure of the owner, slice 6a: broken, by reload or retry only) ``` @@ -579,11 +626,11 @@ host.serve(Notes, ({ session }, principal, self) => - **Activation** (one fresh attempt each time, bounded by `readyTimeout`, its wait for the previous instance's teardown included): the factory runs; singleton documents take `initial()` and, on the object's first activation, persisted documents load and its tables' definitions are checked and migrated (slice 6a; the deadline paused during - eager migration work); references are bound (before the - previous attempt's are released, so a dependency held only by it never ends in between) and waited for until live - and hydrated; natives open one after another; the returned members are checked (section 5); derives compute; - routines start (slice 5). The object goes live when all of that succeeded. Calls wait behind its **gate** until - then. + eager migration work); its requests are made (`self.get`, `self.registry`, keyed pins), and the attempt is + **parked** until every one is live and hydrated (below; the deadline paused); then its own steps run under the + deadline: natives open one after another, followed sources attach, the returned members are checked (section 5), + derives compute; routines start (slice 5). The object goes live when all of that succeeded. Calls wait behind its + **gate** until then. - **The seal fence** (owner, 2026-10-26): every attempt has a seal token (a job run invocation its own, for its state changes and outcome). After the seal, new work through the attempt's handles is refused at use (`self.tx`, `ctx.get`, a job or routine start); commits are checked when they enter the store's commit step (sealed: rejected `released`, @@ -602,20 +649,39 @@ host.serve(Notes, ({ session }, principal, self) => - **A live object can fail** (owner, 2026-10-26): a runtime failure of the owner (a `migrate` failing on access, a stored version newer than the code) seals the live attempt and makes the object `failed` (`broken`): calls reject with the cause, state members keep their last values, natives close, documents stay on the object; recovery by - reload or `ref.retry()`; live dependents stay live. The members freeze only after the commits admitted before the - seal adopted and published once, and a retry's factory waits for that (2026-10-27, slice 6a build reviews). -- **Waiting for dependencies (R18):** an object whose references are not live yet stays `loading` and goes live as - soon as they are (bounded by `readyTimeout`; a link drop while a dependency is still loading is no failure: the - reference re-holds on the next link and the dependent keeps waiting, design audit 6, 2026-10-27). A dependency - going live while the dependent still activates is remembered and retries it at once if that attempt fails. A - dependency that fails (`broken` or `transient`), or does not become - live in time, fails the dependent **`transient`** with the dependency's cause inside, never `broken`: the cause is - the dependency's. The dependent keeps its references while failed, and Chord installs their state as an automatic - retry trigger: the moment the dependency goes live, the dependent is retried at once (the remaining backoff is - cancelled; the backoff stays as a safety net). Factory authors write `self.get` and know nothing of this. When a - dependent's last reference goes, it ends and releases its dependencies. After `live`, a dependency leaving `live` - does not change the dependent: it stays live, and its calls that use the dependency get its error (or park while it - is loading). Only R18 failures keep references; any other failure releases them. + reload or `ref.retry()`; live dependents stay live. Its references keep following every member (its own and + re-exposed ones) until the commits admitted before the seal adopted and published once, then take their last values + (audit A10, 2026-10-30: one rule, as restarts), and a retry's factory waits for that. +- **Parked on its dependencies** (dependency model §3, slice 7a; supersedes R18's dependency deadline and backoff): a + dependency is what an attempt declares and needs before it goes live: `self.get` (local or remote), `self.registry`, + keyed pins (`at(key)` on another object's keyed member); `self.follow` is the attempt's own step, not a dependency. + While any dependency is not live (and hydrated), the attempt is parked and its object shows: + + | Its dependencies | The object shows | + | --- | --- | + | any `loading`, none failed | `loading` | + | any `failed` | `failed { waiting }`, `retryAt` undefined, error `unavailable` "waiting for X: " (X the first failed one in declaration order, the root cause found through its message's "waiting for …: "; always these two, so a ring of waiting attempts settles after one round; within 500 characters, X's label clipped first, fix round 1) | + | one failed with `invalid/cycle` (a dependent off the ring) or `forbidden` (an `authorize` that refused it: authority, M§3.6) | `broken` with that error: nothing heals it but `ref.retry()` or a reload | + | one `ended` (revoked, its host closed) | `broken` with that error: the request cannot come back | + + **Nothing times out while parked:** no dependency deadline, no backoff (only `parked_long`'s diagnostic report, below); the attempt's `readyTimeout` deadline is paused + (as eager migrations pause it) and resumes with the budget left; the identity backoff stays for the object's own + `transient` failures only. **Wakes:** the parked attempt checks again at every change of a dependency (live, + loading, failed, a new cause compared by value, ended, and its first outcome, below), coalesced per object after the + batch that caused them (a change while it checks or loads pinned keys is remembered); it never re-runs its factory. + A parked attempt keeps its documents, declarations, requests and seal token, and holds no natives. **The dependency + check is the boundary:** once all are live (and the pinned keys loaded), the own steps run; a dependency that leaves + `live` afterwards changes nothing (R18 after live: the object goes live; its calls through that reference meet its + state). **Retry passes on:** `ref.retry()` (or a `retry` message) on an object parked `failed { waiting }` passes on + to its failed dependencies (audit A5, 2026-10-30): on one host one retry's walk keeps a visited set, so a diamond + retries a shared dependency once; across links the `retry` message carries a hop budget (`hops`, 16 from the first + host, one less per host, dropped at 0), so a waiting ring across hosts passes it round until the budget runs out + (starting no attempt) and a diamond across hosts may retry a shared dependency twice (harmless). A `broken` or + `transient` one starts an attempt, a parked one passes it on, a request waiting unbound in an index is left as it is + (an event wakes it), a refused hold over a link re-runs its start. A `retryOn` wake while parked does nothing. When + a parked object's last reference goes, it ends at once (it is not live: no `linger`) and releases its requests. + `broken` is only for an object's own code, cycles and authority; no failure keeps references (the R18 retry trigger + is gone: parking replaces it). - **Dependency cycles are rejected (Q5):** every object instance has a uuidv7 instance id, minted at its first attempt; an object that restarts gets a new one, so an old chain never matches (slice 4b, the first user; D4, owner, 2026-10-24). On one host, at @@ -624,34 +690,42 @@ host.serve(Notes, ({ session }, principal, self) => (bounded; ignored from untrusted links): the provider rejects the hold when the instance being activated is in the chain (slice 4b; on one host slice 1; at most 32 entries; locally created objects inherit the activating chain). A cycle fails the activation `broken invalid/cycle` with the cycle as cause ("A → B → A"), and every other activating - object on it the same way; no references are kept. A dependency failing with `invalid/cycle` fails its dependent - **broken** with the same error, an exception to R18 (owner, 2026-10-24: otherwise every backoff retry hits the same - cycle); dependents that are not on the cycle break too, and a reload or `ref.retry()` recovers them. Checked again - after a reload. Cost: O(chain) per declared reference, only during activation. - **Known limitation** (owner, 2026-10-24): a chain is fixed when its object is created, so a cycle entering an object - that is already activating is not detected. Example: A on the server and B on a worker start at the same moment, A - declares B and B declares A; the holds carry `[A]` and `[B]`, neither finds the other, both fail `transient` at - `readyTimeout` and retry with backoff (later attempts usually come in order and detect it). A probe along waiting - holds was not built. + object on it the same way; no references are kept. A dependency failing with `invalid/cycle` makes its dependent + **broken** with the same error (owner, 2026-10-24: it would wait on a ring that never heals); dependents that are not + on the cycle break too, and a reload or `ref.retry()` recovers them. The walk on one host follows the waiting edges + of activating and parked attempts (exact). Checked again after a reload. Cost: O(chain) per declared reference, + only during activation. + **The gap: rings across hosts started at the same moment** (owner, 2026-10-29: no probe): a chain is fixed when its + object is created, so a cycle entering an object already activating is not detected. Example: A on the server and B + on a worker start at the same moment, A declares B and B declares A; the holds carry `[A]` and `[B]`, neither finds + the other, and without a dependency deadline both stay parked. It is made visible: an attempt parked longer than its + `readyTimeout`, whatever its blocker (audit A11, 2026-10-30), is reported once per parking episode as + `parked_long { object, waitingFor, state, since }` (`waitingFor` the blocker's label, `state` `loading` or `failed`); nothing fails or retries. A cycle is a plugin bug; **the guide's four + rules** for plugin authors and agents (host README): (1) a factory declares only what the object needs, never a type + that needs it back; (2) back-references and occasional lookups go into methods and routines (`ctx.get`); (3) data + flows up through state and registries, not mutual `self.get`; (4) host-lifetime objects on the server never declare + worker types in their factory. - **Failed:** a broken part fails the whole object: a throwing factory, document hydration or `migrate`, a failed - open, a derive that throws on its first computation during activation, or a dependency that did not become live. - After the object is live, a throwing derive fails only its own member (`broken`, W1). The cause is tagged and decided by Chord from where it - failed: + open, a derive that throws on its first computation during activation. After the object is live, a throwing derive + fails only its own member (`broken`, W1). The cause is tagged and decided by Chord from where it failed: ```ts - cause: { kind: "transient"; error: ChordError } // an open threw, a reference was unreachable: automatic backoff - | { kind: "broken"; error: ChordError } // factory, derive, hydration or migrate threw: no automatic retries + cause: { kind: "transient"; error: ChordError } // the object's own effect failed (an open, a follow): backoff + | { kind: "broken"; error: ChordError } // its own code (factory, derive, hydration, migrate), a cycle, authority + | { kind: "waiting"; error: ChordError } // not its fault (slice 7a): no provider, no fitting version, a + // dependency failed; retryAt undefined, woken by an event ``` Only an `open` may choose `broken` itself (`throw broken(error)`, e.g. invalid configuration). **Transient** causes get a **backoff per identity** (1 s doubling to 60 s; a map holding no object, so a failed object is collected after its last reference while its window runs; concurrent gets share one attempt; success resets). **Broken** ones are - retried only by a trigger. Triggers (for both): `ref.retry()`, a plugin reload (slice 7) and wake signals the factory + retried only by a trigger. Triggers (for both): `ref.retry()`, a plugin reload (slice 7b, §21) and wake signals the factory declares with `self.retryOn(...readables)` (slice 8; anywhere in the factory: the factory specifies, Chord installs the watches before natives open, so a failed open still has them). Every trigger works like `ref.retry()`: the backoff window is skipped, an attempt starts now, holders are told `loading`. `retryOn` sources must live independently of the attempt (registry values); a reference's member is a type error. A facet cannot fail - after `load`: its setup only declares (a throwing setup rejects `load`, slice 2; owner, 2026-10-26). + after `load`: its setup only declares (a throwing setup leaves its plugin out of `load`, reported in `load`'s + `failed`; decision "Per plugin at startup and reload"; owner, 2026-10-26, 2026-10-30). ### The factory principle @@ -674,8 +748,9 @@ host.serve(NoteStore, ({ session }, self) => { routines and jobs. - Handles stay valid for their attempt's life (owner, 2026-10-26: documents and the line belong to the object; each attempt's handles carry its seal token and definition version); nothing is "read at use" or re-fetched. -- On a reload the factory runs again for pure objects (new methods and derives, same documents); native - objects restart (section 21). +- On a reload the factory runs again: in staging against a recording `self` (declarations only), then for real; pure + objects swap in place (new methods and derives, same documents), native objects restart after their drain, + objects of a removed type end and their holders wait (section 21). ### Pattern: shared store, per-principal views @@ -713,22 +788,56 @@ location are dropped. | a call | `await ctx.get(Type, params, { readyTimeout? })` | when the handler settles | | outside | `await host.get(Type, params, context, { readyTimeout? })` / `host.hold(Type, params, context, { readyTimeout? })` | `release()` | -- **`host.get`** (and `ctx.get`) resolves with a reference once its object is live and **hydrated** (every state member - of its variant has a value). A broken failure rejects at once; through transient ones it waits up to its - `readyTimeout`, then rejects with the last failure's error; a getter that gives up releases its reference. - **`host.hold`** returns a reference at once that follows its object until released (loading, live, failed and - retried as its object is, loading again while its link is down) and never rejects; invalid params (checked at once, also before `load`: design audit 7, 2026-10-27) and a local type - not served here throw at once (before `load` resolved, these fail the reference broken once it did); its `variant` is - `undefined` until its first binding (slice 3b). There is no `state: false` option (C8) and no `optional` flag. +- **`host.get`** (and `ctx.get`, `host.registry`) resolves with a reference once its object is live and **hydrated** + (every state member of its variant has a value). It waits while the reference is `loading`, `failed { transient }` + or `failed { waiting }`, up to its `readyTimeout` (default 60 s, the caller's choice), then rejects with the last + cause; only `failed { broken }` rejects at once (owner, 2026-10-29, decision "Getters wait through `waiting`"). A + getter that gives up releases its reference. **`host.hold`** returns a reference at once that follows its object + until released (loading, live, failed and retried as its object is, loading again while its link is down) and never + rejects; only invalid params throw (checked at once, also before `load`: design audit 7, 2026-10-27). A type not + served where it goes is `loading` before the host's publication and `failed { waiting }` after it, before and after + `load` alike (§3; slice 7a). Its `variant` is `undefined` until its first binding (slice 3b). A UI holds and shows + `ref.state`; code that needs the object now gets it, with the patience it passes. There is no `state: false` option + (C8) and no `optional` flag. - **Narrowing by variant:** `ref.variant` is the discriminant (C15: no `inVariant`/`expectVariant`); `RefIn` narrows to some variants (strict unions, never weakened). -- **Members across attempts** (2026-10-27, slice 6a build reviews): on the same host a reference's state member - follows its object's member across attempts, as a remote reference re-holds: one member source per object and - member name; when a new attempt goes live every member re-binds by name (watches get one diff frame from the last - value, consumers' derives recompute, also one that failed on the member's earlier error, which heals; a member the - new variant lacks fails `invalid/member`); while the object is - failed or activating again after having been live, reads and new watches return the last value and get no frames. - A re-exposed member freezes at the re-exposing object; the provider's own source is untouched. +- **References are the only indirection** (owner, 2026-10-29; supersedes 6a's member source per object and 7b's + hand-over): a consumer's watches and derives over `ref.member` attach through the reference's member, which passes + the subscription straight through to the object's own document or derive (no extra hop, no copy: the commit's value + and ops by reference). When what the reference points at changes (a new object instance after a factory re-run: + reload, retry, restart; a new object after its type was removed and served again; another variant), the reference + moves its subscriptions to the new one and sends one frame, a diff from the last value (consumers' derives recompute, + a failed one heals; a member the new variant lacks fails `invalid/member`). While it points at nothing, its members + keep their last values (a pointer, no copy): while loading (a restart, a retry, a swap whose new instance is not live + yet) its watches get no call (`ref.state` says `loading`); while failed (a live failure, its version refused, its type + removed, a failed dependency) they get one `failed` call with the cause's error and stay open (decision "Failures are + watch calls"); a reference that never had a value reads `not_ready` while loading and its failure's error while failed, and its + watches wait for its first values. A member the new variant lacks is not a failed reference: it fails `invalid/member` + (reads throw it, watches get one `failed` call) until a later variant brings it back. Keyed watches and pins + re-pin on each new instance the same way (a key keeps its last value until the new instance has loaded it); a + dependent's factory pin is still loaded before the dependent goes live. Once hydrated, a reference has last values for every state member of its bound variant, read or not: when it + detaches it takes them at its version from the outgoing instance (through that instance's mapping for an older + version), keeping no node of it. After a `failed` call, the next good value arrives as `[["r", value]]` even when + unchanged; a watch opened while failed starts at the last value and gets the failure once. No source, mapping, + document handle or pin of an instance outlives it through a reference: a detached reference keeps only plain last + values and its failure; releasing it drops them and its bindings; stopping a keyed watch drops its pin. Derives over a failed reference member compute on its last value (a contribution built on it stays); only the + member's own watches get the `failed` call, and `ref.state` shows the failure (owner, 2026-10-29). A re-exposed member is + the re-exposing object's reference member, followed the same way. Over a link the same rule: the replica keeps its + last values readable while its hold is failed. Nothing outlives its object instance and nothing is handed between + objects. +- **References across a reload** (slice 7b §6): a reference is never ended by a code change. Its object swapped in + place or restarted: it follows the new instance (above; a hold over a link gets a fresh `bound`, §22). Its version no longer accepted, or a member of it refused (`contract_mismatch`): it leaves the object and waits + unbound (`failed { waiting }`) with its last values; the object and its other holders stay. Its type removed: it + waits unbound `not_served` with its last values; a publication that serves the name again binds it to the next + object (a new object: its in-memory documents start over from `initial()`, persisted ones load again). On a removal + the references take their last values at the seal: a commit admitted at that moment is stored and loaded by the next + object, but not in the last values they show while they wait (accepted: deferring it would need new state on the + waiting path, owner, 2026-10-29) (one frame, watches and derives heal, keyed pins re-pin, parked calls flush or reject, presence lists it + again). A reference of an older version a provider accepts reads its state members through the object instance's + shared mapping for its version (and keyed members per key), and its calls are mapped forwards and their results + back, or run the provider's replacement of a removed method (`checkArgs`). A reference knows its version from the + contract token it was made with; one made by outside code keeps the token its caller passed (Chord reloads no caller + code). - **Members:** each state member of a reference is a watchable of the provider's member (a replica over a link); watches taken through a reference belong to it and end when it is released. Methods are functions (`ref.add(args, ctx)`, `ref.bump(ctx)`). @@ -736,7 +845,8 @@ location are dropped. ```ts | { kind: "loading" } | { kind: "live" } - | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // retryAt undefined: broken, no automatic retry + | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // retryAt: a transient cause's backoff; + // undefined for broken and waiting | { kind: "ended"; error: ChordError } // released by its holder, or its object ended ``` @@ -744,13 +854,15 @@ location are dropped. | --- | --- | --- | | link drop | park until reconnect | readable (`loading`) | | object loading or restarting | park until live | readable (`loading`) | -| rollout: provider ahead or installing | park until installed, with a deadline (slice 7) | readable | +| unbound (not served, another version) or parked on a failed dependency | reject at once with the cause | `failed { waiting }` (slice 7a) | | object `failed` | reject at once with the cause | `failed` with cause and `retryAt` | -- A failed reference stays alive: while held, Chord retries per the backoff; `ref.retry()` tries at once (a Retry - button; over a link the `retry { hold }` message, which also re-runs a refused hold's start: authorize, placement, - binding; design audit 2026-10-27); success goes `loading`, then `live`. A provider tells a pending - hold's consumer each failure (`failed`, with the rest of its backoff window) and the next attempt (`loading`). +- A failed reference stays alive: while held, Chord retries a `transient` failure per the backoff and wakes a + `waiting` one at the event it waits for; `ref.retry()` tries at once (a Retry button; over a link the `retry + { hold, hops }` message, which also re-runs a refused hold's start: the host address, binding; design audit 2026-10-27; + on a parked object it passes on to the failed dependencies, §6); success goes `loading`, then `live`. A provider + tells a pending hold's consumer each failure (`failed`, with the rest of its backoff window for a transient one), + each new cause of an object parked on a failed dependency, and the next attempt (`loading`). - **Parked calls** (slice 3b): calls made while the reference is loading are queued in order, each waiting at most the reference's `readyTimeout` from the moment it was made (never renewed), then fail `unavailable/park_timeout`; they are sent in order when it goes live (a new call never overtakes them) and rejected with the cause when it fails. @@ -1301,7 +1413,8 @@ self.onLive(async (ctx) => { // set up, wait have finished (bounded; a stuck call is reported and the new routines start anyway): a handover per object, so routines need no names. Example it prevents: an old handler midway through `restart(W1)` and a new one seeing W1 still pending and restarting it again. The bound is `closeTimeout` (owner, 2026-10-24); calls do not wait for the - handover, only the new routines do (slice 7). + handover, only the new routines do (slice 7b). A stuck old call past the handover is reported `drain_timeout` and + the whole old attempt is sealed then (audit A12: one seal token per attempt; its method calls had the same bound). - **`on(source, fn)`:** any watchable (a document, a derive, a reference member, a registry value, `ref.state`). The first call gets the current value (`[["r", value]]`, W2b), then one call per change; calls never overlap (S8); changes made during a call arrive folded in the next (newest value, folded ops); a running call is not aborted by a @@ -1413,7 +1526,7 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi | { readonly kind: "threw"; readonly error: ChordError | PluginError } // run threw (reported) | { readonly kind: "aborted"; readonly reason: "abort" | "reload" | "restart" } | { readonly kind: "restart_limit"; readonly restarts: number }; // persisted runs (slice 6b) - // abort(id); a native restart on reload (slice 7); "restart": a persisted run abandoned after a crash (slice 6b), + // abort(id); a native restart on reload, in-memory runs (slice 7b); "restart": a persisted run abandoned after a crash (slice 6b), // or a run cut off by its attempt's seal while the object continues (slice 6a) ``` - **Abort is a request:** `abort(id)` fires `run`'s `ctx.abortSignal`; `run` decides what it returns (a result that @@ -1438,13 +1551,23 @@ host.serve(ChatStore, ({ conversation }, self) => { // local: no princi the abort signal, the drain is bounded by `closeTimeout`; an in-memory run's outcome and state go with the object (no `finish`). Jobs in `local` stores are never revoked (no principal): a store that runs work for a principal keeps the grant as data and reconciles with a routine (§25). -- **Versions (`accepts`, slice 7):** a job is versioned like everything else; `accepts` maps an older version's args +- **Versions (`accepts`, slice 7b §8):** a job is versioned like everything else; `accepts` maps an older version's args forwards and its result forwards (`{ 1: { args: (old) => current, result: (old) => current } }`; a producer's - direction, unlike a contract's). A run started on an older version finishes on its old code across a pure reload; - `finish` and `running` see its args and result mapped (mapped once per run, never per publication); its state - document migrates through its own `migrate` (when it may migrate while the old code still writes it is slice 7's - rule). Without an adapter, or for a run of a newer version than the code (a rollback), the run is - `blocked { version }` until a reload brings matching code. A version change aborts nothing and discards nothing. + direction, unlike a contract's). Its records keep every run's version: a new attempt's `self.job(Def, impl)` at + another version takes the records over (the same version with another fingerprint, over args, result and state, is + refused at declaration, as documents). A run started on an older version and still running finishes on its old + code across a pure swap (its invocation keeps the old implementation); `finish` runs on the new code with the + outcome's args and result mapped forwards (once per run, never per publication); `running` shows its args mapped; a + run not invoked yet runs on the new code with its args mapped. Its state document keeps its version while the old + code writes it and migrates through its own `migrate` once its run settled or moved to the new code (the outcome + `finish` gets). Without an adapter, for a run of a newer version than the code (a rollback), or for a job the new + code no longer declares, the run is `blocked { version }` (kept, not invoked) until a reload brings matching code; + blocked runs (`finish_failed` ones too) are retried on every reload of their object's plugin; a persisted run of + another version or fingerprint stays blocked with its record as stored until the reload that brings its version + resumes it. A native restart aborts running in-memory runs (`aborted { reason: "reload" }`) and `finish` runs on the + new code; persisted runs are stopped and resumed per their `resume` (6b R3, owner, 2026-10-29). A `finish` in flight + across a reload completes on the code it started with; the next one runs on the new code. A version change aborts + nothing and discards nothing. - **Persistence (slice 6):** `persist: { resume: "rerun" | "abandon" }` keeps the run record and its state document in the owner's store (owner: a `local` object with empty params, P3). How a run resumes after a crash is required, no default: rerun (the current @@ -1485,7 +1608,7 @@ export default defineFacet((host, self) => { // `self` (contributions) a - A plugin's per-host setup, **declarations only** (owner, 2026-10-26): `host.serve`, `self.contribute` (slice 8), `host.references`; no natives, no documents (C16 accepted), no routines, no jobs. Everything a plugin does at runtime is done through objects: shared or persisted state and standing behaviour live in a `local` store object with empty - params (host lifetime, its own failure domain, R18 recovers its dependents), as p2 does (its 63 facets take only + params (host lifetime, its own failure domain; its dependents park while it is failed and go on when it recovers), as p2 does (its 63 facets take only `host`). A facet cannot fail after `load`. - `host.serve(Type, factory, { linger? })`: `linger` (milliseconds, 0 by default; slice 3b) keeps a live object with params alive that long after its last reference, so a new reference within that time reuses it with its documents @@ -1505,7 +1628,7 @@ host.serve(Http, { // the defining p accepts: { 1: { contribution: (old: HttpContributionV1) => upgrade(old), value: (router) => routerV1(router) } }, }); self.contribute(Http, { routes: { … }, wraps: [ … ] }); -const router = self.registry(Http).value; // live (host.registry(Http) outside objects) +const router = self.registry(Http); // a declared dependency; read router.value at use (await host.registry(Http, ctx) outside objects) ``` - Each contributor (a facet or an object) contributes **at most one value per registry**; the table is an immutable, @@ -1518,9 +1641,15 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi `compose(table, { report })` is pure and runs whenever the table changes (after a commit, when a contributor goes live or ends, when a dynamic contribution changes). It always delivers a value with its fallback built in (e.g. the first by plugin name wins) and reports conflicts through `report` (owner notices and telemetry events); readers get - a plain value and never handle conflicts. Only if `compose` throws is the registry in an error state, and readers - get an error when they use it. Work that started earlier keeps the value it read (a pinned Durable phase keeps its - old table). Helpers: `keyed(table, entriesOf)`, `ordered(table, itemsOf, orderOf?)`. + a plain value and never handle conflicts. If `compose` throws, the failure is reported and its + watches get a `failed` call, while `.value` keeps the last good composition (slice 8), **but only while every + contribution it was built from is still in the table** (dependency model decision 5, slice 7a): when one leaves (its + contributor ended, failed or restarted), the last good value is dropped and `.value` throws until `compose` + succeeds, so it never hands out a native whose contributor ended; only before any good composition, or after such a + drop, do readers get the error. Work that started earlier keeps the value it read (a pinned Durable phase keeps + its old table). Helpers: `keyedEntries(table, entriesOf, report)` (a map, the first by plugin name wins, a duplicate + reported), `orderedEntries(table, itemsOf, orderOf?)`; both keep each item's plugin (named so beside the contracts' + `keyed`, slice 8 build). - **Who contributes (R21):** only facets and host-lifetime objects (`local`, empty params); their factory's `self` has `contribute`, a referenced object's `self` does not (a type error). Reading a registry's composed value stays open to every object. - **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an @@ -1528,9 +1657,11 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi - **The owner decides** conflicts, order and selection (which contributions apply where). - **One authoritative declaration (Q8):** only the facet of the plugin that defines a registry declares it to the host (`compose`, version, `accepts`); other plugins' tokens only identify it by name and types (contribute and read, - never bring their own `compose`). A reload of the defining plugin replaces the declaration atomically and the table + never bring their own `compose`). A reload of the defining plugin replaces the declaration atomically (`serve` + replaces; a removed defining plugin's provider goes, its readers wait `not_served`) and the table recomposes. Without a declaration (the defining plugin is not loaded) the registry has no composed value: readers - get an error like a failed reference; contributions wait in the table. + wait `failed { waiting }` (`unavailable/undeclared`), like a request for a type nobody serves; contributions wait in + the table. - **Versions (Q8, the owner-adapter pattern):** a contribution carries the registry version its plugin was built against; older ones are adapted with the defining facet's pure `accepts` when the table is built. A contribution without a matching version or adapter (older without adapter, or newer) is left out of the table and reported (an @@ -1540,12 +1671,16 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi use site catches synchronous throws and rejections when it calls an entry, reports naming the contributor (the composed table keeps each entry's plugin name), and continues with the other entries. Durable already does this for extensions (`extension_error`; a throwing tool becomes an error result); `compose` reports through `report`. -- **Slice 8's rules** (decisions "Slice 8 D1" to "Slice 8 S3", `slice-8.md`): the token is shared, the provider serves it - (`host.serve(R, { compose, accepts })`); `self.registry(R)` in a factory is a declared dependency (activation waits - for a value; with none after `load`, the reader fails `transient` and is retried when one arrives); contributions - never fail a registry (a failed one is left out and reported); a throwing `compose` keeps the last good composition; - older readers get the value mapped back (`accepts[old].value`); `retryOn` retries on value calls only; `host.root(fn, - context, { principal? })` seals its context at settle, abort and `host.close`. +- **Slice 8's rules** (decisions "Slice 8 D1" to "Slice 8 S3", `slice-8.md`; S2's `broken` and S3's backoff superseded + by slice 7a): the token is shared, the provider serves it (`host.serve(R, { compose, accepts })`); `self.registry(R)` + in a factory is a declared dependency: the attempt parks until the registry has a usable value at its version + (composed; mapped by `accepts[old].value` for an older reader), otherwise it shows `failed { waiting }` naming the + registry (`not_served`: `unavailable/undeclared`; `version`: a reader newer than the provider or without a mapping, + `contract_mismatch`; `compose`: a first `compose` or the mapping threw; `loading` before the publication), woken at + every change of whether the value is usable; `host.registry` waits the same way up to its `readyTimeout`; + contributions never fail a registry (a failed one is left out and reported); a throwing `compose` keeps the last + good composition (above); `retryOn` retries on value calls only; `host.root(fn, context, { principal? })` seals its + context at settle, abort and `host.close`. - **Registries may carry natives.** An entry lives exactly as long as its contributor; consumers look up at each use or follow the composed value and keep nothing from an entry beyond a use. - **Contributions that change (owner, 2026-10-19; W4):** registries are host-local, so contributions are usually @@ -1565,6 +1700,9 @@ const router = self.registry(Http).value; // live (host.registry(Http) outsi Tools, sections, hooks, wraps and tasks reach the harness as one Durable `Extension` per plugin contributed to p3's core registry; `compose` builds Durable's registry snapshot (Durable applies selection and wraps, and pins a snapshot per task phase). Skills and other lists work the same way. +- **A contribution may be another registry's value** (it is followed: its change recomposes). A ring of registries + reading each other is caught when one composes (audit A2): a registry read while it composes throws `invalid/cycle`, + the contribution that read it is left out and reported `contribution_failed`, the rest of the registry works. - **Removing a contribution:** there is no remove call (`contribute` is declarative, in the factory). An entry leaves the table when its contributor ends, or after a reload whose new code no longer contributes. A live contributor that has nothing to offer for now sets its readable to the registry's empty value (an extension without tools, empty @@ -1678,89 +1816,203 @@ agent asks everyone present. Its store must live in an object (it predates "nati ### Per host -``` -1. load new bundles; an import or syntax error rejects -2. staging facet setups and every live object's new factory run with a recording `self` (declarations only, - nothing activated); any throw rejects -3. commit the host switches to the new code atomically -4. after per object (below) +`host.reload(facets, context)` (slice 7b; decision "Slice 7b's open points, as recommended" (1)) takes the host's +**complete new plugin set** after `load`: a plugin in it and not loaded is added; one loaded with another facet value is +changed (Chord compares no code); one passed as the very same facet object is unchanged and untouched (owner, +2026-10-29: base code passes its core facet value every time); one loaded and not in it is removed. It resolves with a +report once the commit happened and every object took its fate (swaps and retries had their first dependency check; +it does not wait for objects to go live); a plugin whose new code fails keeps its old code and is listed in the +report's `failed` (per plugin, below). +Reloads run one after another per host; before `load` resolved it throws, after `close` it rejects `released/closed`; +`context` bounds it (decision 4: no install watchdog): its abort before the commit rejects `aborted` and changes +nothing, after the commit it only ends the wait: it rejects `aborted` with `details.committed`, a `CommittedReload` +`{ added, changed, removed, failed }` naming what the commit applied (the plugins component adopts it, so a later +reload never passes a committed plugin's old facet; owner, 2026-10-30). + +```ts +type ReloadReport = { + readonly plugins: { readonly added: readonly string[]; readonly changed: readonly string[]; readonly removed: readonly string[] }; + readonly failed: readonly { readonly plugin: string; readonly error: unknown }[]; // new code failed: old code kept, an added one left out + readonly objects: readonly ({ readonly object: string } & ( + | { readonly kind: "swapped" } | { readonly kind: "restarted" } | { readonly kind: "ended" } + | { readonly kind: "parked"; readonly cause: FailureCause } | { readonly kind: "retried" }))[]; +}; ``` -- **Rollback is all-or-nothing per host:** any load or staging error (code errors: load errors, throwing factories) - leaves the whole host on its previous code; a host is completely at revision R or R−1, so status per host suffices. - Runtime states (registry conflicts, failed opens, migrations) are not rollback reasons. -- **Pure objects** (no `self.open`, old or new) are swapped in place: new methods and derives; documents, references - and jobs stay; routines restart (serialized); new references become live; then the gate opens. Running calls finish - on old code, and keep their old handles valid until they settle (their transactions keep working), except for - documents whose version rose (R15, as in Durable): old code touching a document migrated past its version throws, - the document stays intact, and the call fails once (not resent). Only new calls get the new handles. -- **Native objects of the reloaded plugins restart:** new calls park, running calls are drained first (bounded by - `closeTimeout`; owner, 2026-10-26: a call outliving the drain is cut off by the seal fence), then natives close (reason `reload`), - the factory runs, natives reopen; documents stay. In practice this costs little: the Durable harness is core (base - code), terminal PTYs live in the separate terminal host process (the server's native is only the connection), model - runtimes, sandboxes and the search index rebuild in milliseconds; only reloading the plugin that owns SSH connections - (running remote commands are killed) or the relay socket (relayed connections drop) hurts. Natives are not kept - across reloads by name: natives have no line, no transactions and no schema, so old and new code would interleave - unchecked. -- **Derives on reload:** derives have no names; exposed derives are matched by their member name, internal ones are - rebuilt (matching only if a real need appears). -- **Holds across reloads:** a hold's indexes belong to its bound version. A reload that still accepts that version - keeps the hold: a pure object re-subscribes it through the version's mappings and sends `[["r", value]]` for each - member whose value changed; a restarted object sends a fresh `bound` for the same hold (a `bound` on a bound hold - replaces its replicas). A served hold whose object leaves `live` while its `bound` waits for a slot goes back to - pending (slice 7 builds it: nothing in 4a leaves `live`). A reload that drops the version ends its holds with `gone` (`contract_mismatch`) before - the new code serves anything. A consumer whose own code moves to a new version re-holds (a new hold id) and releases - the old. -- **Old-version calls:** `checkArgs`'s `{ kind: "call" }` result runs the provider's replacement; `{ kind: "method"; - result }` maps the result back for an old-version caller (section 5). -- **Retries:** a reload retries a broken object as `ref.retry()` does (section 6). -- **After the commit,** errors stay per object: a failed open or reference makes that object `failed` (backoff); a - migration that throws on a lazily loaded document refuses that access; a method bug fails that call. -- **Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory - documents go with them; persisted ones stay until retired. -- **Consumers elsewhere** see a restarting object like any object that briefly goes away. -- No factory trial, no migrate dry run, no consumer contract check during staging: incompatible consumers fail at bind - with `contract_mismatch` and a notice naming plugin and type. -- **Start barrier** (host start and first load): load facets (the core facet with its start data among them), wait - until every facet and host-lifetime object is live or failed (an object that must - contribute at start, like the model runtime, is a `local` object with empty params). Then the registries hold the contributions of every - plugin that loaded, and dependent work - resumes: base code calls `harness.resume()` after `load` (the harness is an ordinary host-lifetime object that - opened Durable paused during the barrier; owner, 2026-10-21, refines R17). Nothing - waits for specific contributions: what is missing shows where it is used (section 24). Failed plugins are reported - with their error. Plugin reloads need no barrier. +``` +1. load new bundles (the app's or loader's: a facet arrives evaluated); a plugin whose build or evaluation + fails keeps its loaded facet (`reload(facets, context, { failed })`) +2. staging every new setup into a staged table, and every affected object's new factory against a recording + `self` (declarations only, nothing activated, bound or opened); a plugin whose new code throws keeps its + old code, staging runs again without it, then the closure rule (below) +3. commit one batch: the host switches to the new code atomically, a publication (7a) +4. after each object takes its fate (below) +``` + +- **Per plugin, never all or nothing** (owner, 2026-10-30, decision "Per plugin at startup and reload"; supersedes 7b's + rollback per host): a plugin whose new code fails at a reload (its build or evaluation, a setup's or factory's throw + in staging, a declaration error, `checkAccepts`, a type or registry served twice, a remote type on a host without + principal or address) keeps its old code (its loaded facet, unchanged) and is reported in the reload's report + (`failed: [{ plugin, error }]`, the same list `load`'s report has; an added plugin that fails is left out); staging runs again without it, and every other plugin of the reload applies. **The + closure rule:** a changed plugin whose recorded requests would not bind because the plugin they need was kept old (or + is missing) is kept old too, so a consumer edited together with a broken provider keeps working on its old code. At + startup the same holds: `load` leaves a failing plugin out and resolves with `{ failed: [{ plugin, error }] }`; its + consumers wait `failed { waiting }`; `load` never closes the host for a plugin (base code decides whether its core + failing is fatal). A loader that kept a plugin on its loaded facet because its new code failed before staging (its + build, its evaluation) tells the host with `reload(facets, context, { failed })`: the host lists it and applies the + closure rule to it as to a plugin whose staging failed. What no rule detects: a consumer relying on new behaviour of a kept provider whose contract did not + change (as in a rollout across hosts). Runtime states (registry conflicts, failed opens, migrations) stay per object + after the commit (`onError` as always). +- **Staging** runs every new setup (each plugin into its own table, merged; an unchanged or kept plugin's loaded + entries are reused, its setup not run again) into a staged table beside the live one (types with their factory, + `linger` and provider, `bindProvider(contract, options.accepts)`; registry providers; facet contributions by + plugin), then, for + every live, activating, parked or failed object of a changed or removed plugin whose type is still served, its new + factory with the object's params (and principal) against a recording `self`: it records what the factory opens, + the documents, tables and jobs it declares with their versions, its contributions and requests, runs the checks an + attempt makes without effects (declared twice; persisted data outside a stable identity; contributions outside a + host-lifetime object; a document or job at an older version, or at its version with another schema), and hands out + inert handles. Staging is synchronous, so no request or arriving hold meets it half done (audit A1: model §1's + `installing` is cut). +- **The commit** (one batch): the served types become the staged ones (an unchanged plugin's entries stay as they + are); a removed plugin's registry provider goes (its readers wait `not_served`), a reloaded one's replaces the old + (`RegistryRecord.serve` replaces), the reloaded plugins' facet contributions are replaced by plugin (object + contributions follow their objects), every changed registry composes once; a removed type's presence list goes. + After the batch, each object takes its fate, host-lifetime objects of new types start (no barrier), and the + publication wakes the requests indexed under the names the reload serves (`UnboundIndex.wake(names)`, per name), + refused holds of those names re-run their start. +- **Fates** (decided in staging; objects of plugins not in the reload are untouched): + + | The object | Fate | + | --- | --- | + | its type is removed | **ended**: as an end (natives close `end`, callers of running calls rejected `released`, handlers drained, in-memory documents gone, persisted ones kept); its holders' requests are not ended: they wait unbound `not_served` (§7) | + | failed, parked, activating or waiting for its backoff | **retried** on the new code (`ref.retry()`'s rule, whatever broke it); an activating attempt is sealed and its natives close `reload` | + | live, neither its live attempt nor the new factory opens natives, and no table of the new factory is newer than the store's | **swapped** in place (below) | + | live, otherwise | **restarted**: new calls park (it shows `loading`), its contributions leave, running calls drain (bounded by `closeTimeout`; a call outliving the drain is cut off by the seal fence), the attempt is sealed (in-memory job runs end `aborted { reload }`, persisted ones are stopped and resumed, R3), natives close `reload`, the new attempt activates after that teardown (documents stay; eager table migrations run as at activation) | + | any of the above whose new code waits for a dependency | **parked** (decision 4): no overlap of old and new code; reported with the cause | + + A restarted object gets a new instance id (an old activation chain never matches it); a swapped one keeps it; a + retried one keeps it as any retry does. Cycles are checked again for the new attempts; `parked_long` is unchanged. +- **The pure swap** (decision "The seal fence": not a replacement while its calls run): a new attempt of the same + object (its documents, line and job records) activates at once; calls arriving meanwhile wait at + the gate (bounded by `readyTimeout`, `unavailable/park_timeout`). The new attempt's requests are made and bound + before the old attempt's are released (a request equal to an old one binds to the same target with no gap, M§3.9). + Running calls finish on the old attempt with its handles valid, except R15: an old call touching a document whose + version rose throws once (`released/version`), the document intact, the call not resent; the same for a table whose + version rose (old table handles' version fence: old code never issues SQL for a dropped column). Once the swap + ended (the new attempt live, parked or failed) and the old attempt's calls settled (bounded by `closeTimeout`, + `drain_timeout`), the old attempt is sealed and its references released; its job invocations finish on their old + code. Retiring attempts are bounded by `closeTimeout` each; a later swap never waits for them (several may retire + at once after quick reloads), while a restart, an end and the host's close do (an end rejects their callers at + once). The references move to the new instance's members by name (one frame from the last value; a member the new + variant lacks fails `invalid/member`; §7); exposed derives are matched by their member names, internal ones are + rebuilt; in-memory documents migrate through `migrate` when the new code declares a higher version; the + contributions are replaced in one batch; `self.writable` starts at its new initial value (decision 2); `retryOn` + sources are re-declared. **Routines hand over per object:** the old attempt's routines are aborted; the new ones + start once the old routine calls finished, bounded by `closeTimeout` (a stuck one is reported `drain_timeout` and + the whole old attempt is sealed then, audit A12: one seal token per attempt); calls never wait. +- **Natives are not kept across reloads by name:** natives have no line, transactions or schema, so old and new code + would interleave unchecked. In practice a restart costs little (the Durable harness is base code, terminal PTYs live + in their own process, model runtimes, sandboxes and the search index rebuild in milliseconds); reloading the plugin + that owns SSH connections or the relay socket hurts. +- **No factory trial, no migrate dry run, no consumer contract check during staging:** staging records declarations; + incompatible consumers wait at bind (`failed { waiting }`, `contract_mismatch`). The one exception is the closure + rule above: when a plugin of the reload failed, a changed plugin's recorded requests are checked as a bind would + check them, to keep it old. +- **Start barrier** (host start and first load; dependency model §6, decision (a), slice 7a): `load` runs the facets' + setups (the core facet with its start data among them) and **publishes** them in one batch (types served, + registries served and composed once, contributions in their tables); requests that waited for the publication are + placed after that batch (holds and `host.hold` wait for the publication only, never for the barrier: two hosts + whose objects need each other's types start without a deadlock). Then it activates the host-lifetime objects and + resolves when each has its **first outcome**: live, failed by its own cause, or parked with no blocker that is a + local object still without its first outcome (`local`: executing on this host, a keyed pin still loading here + included; a wait on another host or on a failed dependency ends the first pass), or ended (`host.close` settles a + waiting `load`). One first-outcome signal per object, separate from the readiness dependents wait for, counted once, + never reopened by retries. So `CodeMode → Harness` waits for the harness's outcome. **What it promises:** every + host-lifetime object here had its first try, and the contributions of those that went live are in their tables; + **not** every tool that will arrive (a parked or slow contributor, tools found in `onLive`, MCP): what a first turn + needs is p3's explicit admission check. Base code calls `harness.resume()` after `load` (the harness is an ordinary + host-lifetime object that opened Durable paused during the barrier; owner, 2026-10-21, refines R17). Nothing waits + for specific contributions: what is missing shows where it is used (section 24). Failed plugins are reported with + their error. Reloads need no barrier. ### Rollout across hosts -- A **revision** is (server incarnation, sequence) and names the complete desired code set (every facet with its - digest), not a batch. Every install reconciles everything missing; a host reports a revision only when it runs that - complete set. -- **Order:** server, workers, clients. Each host reports `installed`, `installed_failed` (named failed objects) or - `rejected` (the reason; the host stays on its previous code). A server that rejects stops the rollout. -- A host that starts or reconnects installs the current revision before anything is routed to it. "Installed" means - the code is published; objects activate on demand. -- **Mismatches:** a mismatch response carries the provider's revision and status. Provider ahead, or behind and - installing → `updating` (values readable, calls park, with a deadline); provider rejected → fail at once (retry on - the next revision); same revision → `contract_mismatch`, final. Calls parked on a binding that is released end with - an error and are not resent. No cross-host transaction, no votes. +- **The rollout is the app's** (decision "Slice 7b's open points" (3)): it calls `host.reload` on each host in order, + server, then workers, then clients; Chord guarantees per host (staging, commit, a failing plugin kept on its old + code). The plugins component's `rollout` (slice 7c) runs such steps. An install never waits for activations (a reverse-order rollout would + deadlock). +- **Each host knows only its own install** (model §5): until its reload commits (while the app bundles, before + `host.reload`), a hold that does not fit its running code is answered `failed { waiting, version }` and the commit + wakes it (a blip of at most one install; staging itself is synchronous, audit A1). What a consumer on + another host sees follows §7: kept and freshly bound while its version is accepted; `failed + { waiting }` (`version`, `not_served`) otherwise, bound again by the install that serves it; a consumer whose own + code moves to a new version releases its old request and makes a new one. +- **The app's loader keeps its own duties:** the desired code set, a digest over the facet bundles' content hashes (an + unchanged server restart reinstalls nothing), serialized installs, a host that starts or reconnects installing the + current set before anything is routed to it, a rejected own install failing its own requests with the reason. -- **A host checks its own install status before waiting on a provider:** a rejected own install fails at once with - the reason. **Installs are serialized per host.** **A revision also carries the code-set digest** (the hash over the - facet bundles' content hashes, which loader D computes anyway; no module analysis); hosts compare the digest on - reconnect, so a server restart with unchanged code triggers no reinstall. +### Loading plugins and the reload loop (slice 7c) -### Loading and the dev loop +- **No dev/prod split** (owner, 2026-10-30): a Chord app is self-modifying (the owner or an agent edits a plugin while + it runs), so finding, building, loading, watching and rolling out plugins is core machinery: one component, + `chord/next/plugins` (`src/next-node/plugins/`, Node-side; the kernel stays portable and takes facets however they + were made: static imports in tests and Durable Objects work the same). Reloads stay explicit: the watcher marks, + never applies. +- **A plugin** is a directory with a `package.json` (its `name` is the plugin's name, npm style) and one file per facet + kind the app names; plugins come from roots found by directory listing (no source parsing; the first root wins a + duplicate name). Dependencies between plugins resolve by contract name at runtime; names are prefixed + (`.`; tools `_`). A plugin carries the contracts it was written against (decision "Contracts come from where the + plugin's code came from"): plugins that change together import each other by relative path within a root + (copied into the importer's script); others take contracts as an npm dependency at a fixed version (built in slice 12, + which removed slice 10's imports by plugin name). +- **Loader D:** one esbuild script per facet (its own files, copied files of other plugins, its npm `dependencies` + inlined; host packages, `chord.external` packages and Node builtins external, passed in at evaluation); each facet kind is defined once, `kinds: { : { platform: "node" | "browser", hostPackages } }` (slice 11); evaluated with + `vm.Script` under a stable filename per (plugin, kind, digest); no module map: old generations are reclaimed by the + GC; V8's compilation cache turned off when the component is created. `pluginFile(path)` resolves a file of the + calling facet's own plugin. A plugin's dependencies are installed into its directory (`installDependencies`, as + coding-agent does: `--omit=dev --legacy-peer-deps --ignore-scripts`). esbuild is an optional peer dependency. +- **The reload loop:** a watcher marks plugins dirty from the files each facet used (a copied contract dirties every + user) and flags changes of base code (the app restarts its hosts); a build is per plugin (a failed one keeps its old code); `rollout` runs the app's + steps in order (server, workers, clients; decision "Slice 7b's open points" (3)). A reload control (a button, a CLI, + an agent tool) is the app's. One base bundle per surface (host packages and the app runtime) is an app's startup + optimization, the web's in slice 11. +- Project plugins, trust, snapshots, scopes and npm/git package installs: `apps/p2/docs/plugin-loading.md` (still the + reference for those; not in 7c). -- Plugins come from built-in, global (`/plugins`), package (npm/git, installed with `--ignore-scripts`) and - project roots, found by directory listing (no source parsing); the server selects; dependencies resolve by contract - name at runtime; names are prefixed (`.`; tools `_`). -- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet (host - packages passed in, no module map); old generations are reclaimed by the GC; stable filename per (facet, digest) and - V8's compilation cache off on hosts that reload in place. The loader is an adapter: static imports (tests, Durable - Objects) work the same. -- Project plugins, trust, snapshots and scopes: `apps/p2/docs/plugin-loading.md` (still the reference for those). -- **Dev loop:** a source watcher marks plugins dirty (never applies changes); a reload control (playground web shell, - CLI, agent tool) reloads the changed plugins or restarts all hosts for base code. +### The plugin model (2026-11-01; slice 12) + +Decisions "Contracts come from where the plugin's code came from", "Plugins import only the app's versioned API", +"The plugin model, points 3, 4, 5 and 8, imports and builds", "Single-file plugins", "Testing a plugin alone". The +rule underneath: a plugin carries what it was written against, never the running app's newest file. + +- **What a plugin may import:** its own files; the app's versioned API (`@acme/app/v3`, never Chord directly: Chord is + not a host package plugins may name; the API re-exports the Chord surface the app supports); its npm `dependencies` + (bundled at the installed version); other plugins' contract files: by relative path within one repository + (`../notes/contract.ts`, the same commit holds both sides), by package name across repositories (`@bob/notes`, + `@bob/notes/web`, pinned by an npm version or a git ref `github:bob/notes#`; a plugin inside a + multi-plugin repository through a dependency on the whole repository, by path into it or its `exports`). A package + lists the plugins it provides in `chord.plugins` (as pi's manifest). One facet imports one API version only. +- **Versions:** the app lists the API versions it provides; an unlisted one is a build error naming the available ones. + The process runs one Chord: an older API version is a layer over today's, re-exporting Chord's compatibility module + for its previous API while Chord keeps one; contracts bind through `accepts` whatever Chord runs. +- **Split by host kind:** the API has a shared module and per-kind modules (`/v3/web`, `/v3/tui`, `/v3/server`, + `/v3/worker`); a plugin's contract files mirror it (`contract.ts`, `contract.web.tsx`, `contract.tui.ts`, ...). A + contract file mentions native things only through `import type`. The build checks every import by the file it + resolves to: a per-kind file or module only for its kind, a package the kind does not provide refused. A token or + `local` contract used on one kind lives in that kind's file. Shared UI between plugins: an imported file (inlined, + frozen at the importer's build) or a registry entry (one live instance). +- **Base code** is the API modules, bootstrap (hosts, store, components such as `createHttp` and `createWorkers`, which + take settings and process values and return facets) and one start-data facet serving process values as core + objects. Every implementation, the harness and core registry providers included, is a plugin; the app's own root is + required (a failure at startup stops the app). +- **Single-file plugins:** `..ts(x)` at a root's top, grouped by name, no `package.json`, importing only the + API; a directory of the same name replaces it as the same plugin. +- **Installs** from git (a branch, tag or commit cloned into a root) or npm. **Builds** are one esbuild call per facet + from source (TS, TSX, JS, JSON, CSS, assets and WASM by URL); plugins bring no build hooks. +- **`testPlugin(dir, { kind, modules, stubs, start?, principal? }, ctx)`:** the plugin built as in production on an + in-memory host and store, its dependencies served by stub facets, contributions to registries collected for the + test, the contracts nobody serves listed. - **UI reload vs. leaving:** "leaving" is never derived from an unmount, only from explicit navigation or presence (a dropped reference); a component unmounted by a reload has not left. @@ -1783,31 +2035,32 @@ validation, no clone of snapshots or ops. - **The protocol version** travels in `hello`; hosts of different versions refuse each other (a web page reloads for a newer server). Error reasons and message kinds are part of it. Until p3 ships, changes keep `PROTOCOL_VERSION` 1 (nothing is deployed); after that every message or reason change bumps it. -- **One typed `Message`** (owner, 2026-10-22, "Wire protocol"): a TypeBox schema `MESSAGE`, a tagged union over +- **One typed `Message`** (owner, 2026-10-22, "Wire protocol"): a pi-schema schema `MESSAGE`, a tagged union over `kind`, from which the protocol types derive; every decoded message is checked against it (compiled once) unless the link is trusted (section 3). Kinds: | Kind | Direction | Fields | | --- | --- | --- | | `hello` | both, first | `protocol`, `incarnation`, `calls` (the call window N) | - | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address?`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | + | `hold` | to provider | `origin?`, `hold`, `type`, `version`, `fingerprint`, `params` (part), `address`, `principal?`, `incarnation?`, `chain?` (activation chain, Q5), `limit?` (decoded size limit, 4b) | | `revoke` | parent to child (route link) | the revocation value (4b) | - | `release`, `retry` | to provider | `origin?`, `hold` | + | `release` | to provider | `origin?`, `hold` | + | `retry` | to provider | `origin?`, `hold`, `hops` (the retry's hop budget, 1-16: 16 from the first host, one less for each host it passes on from, none sent at 0; audit A5); relays forward it unchanged | | `call` | to provider | `origin?`, `call`, `hold`, `method` (index), `args?` (part; absent: no arguments), `segments?` | | `abort` | to provider | `origin?`, `call` (named `cancel` until 2026-10-27) | | `ack` | to provider | `origin?`; one per applied `bound`/`state` message | | `closed` | relay to provider | `origin`: the downstream origin ended | | `bound` | to consumer | `origin?`, `holds: [{ hold, variant, members: [{ member, value }], failed }]`; one batch | | `state` | to consumer | `origin?`, `holds: [{ hold, members: [{ member, ops }], failed, retired }]`; one batch (`retired`: members retired since the last message, slice 6a) | - | `failed` | to consumer | `origin?`, `hold`, `cause { kind, error }`, `retryIn?` (absent: broken) | + | `failed` | to consumer | `origin?`, `hold`, `cause { kind, error }` (`kind`: `transient`, `broken` or `waiting`, slice 7a), `retryIn?` (a transient cause's backoff; absent: broken, waiting); each new cause of a parked object is a message of its own | | `loading` | to consumer | `origin?`, `hold`: a new attempt after a failure, or a relay's upstream dropped | | `result` | to consumer | `origin?`, `call`, `value?` (part; absent: `null`), `segments?` | | `error` | to consumer | `origin?`, `call`, `error` (a ChordError's JSON or a plugin's code, message, details) | - | `gone` | to consumer | `origin?`, `hold`, `error`: the held object ended (revocation 4b, reload 7; relays never make one) | + | `gone` | to consumer | `origin?`, `hold`, `error`: the held object ended (revocation 4b; a reload never ends a hold, it unbinds it with `failed`, slice 7b; relays never make one) | | `ping`, `pong` | both | the heartbeat | A `bound` on a hold that is bound already replaces its replicas in one batch, another variant allowed (re-holds, - slice 7's restarted objects). + slice 7b's restarted objects and its swaps to another variant). - **Codecs are pluggable per link** (both ends use the same): `binaryCodec` (default: a kind byte with a flag for a following origin varint, varints, raw incarnations and fingerprints, length-prefixed strings and parts; about 23 B for a streamed token) and `textCodec` (Delta-style JSON tuples without field names, a message with an origin wrapped @@ -1820,7 +2073,7 @@ validation, no clone of snapshots or ops. before its parts are parsed. The text codec re-serializes parts at decode (one parse per part per hop, accepted for a debugging codec). - **Names travel as indexes** into the hold's bound version (each variant's state members and methods sorted; variants - sorted). A reload that drops that version ends its holds first (slice 7). + sorted). A reload that no longer accepts that version unbinds its holds first (`failed { waiting }`, slice 7b). - **Checks by layer:** the codec's structural decode, the `MESSAGE` envelope (skipped on trusted links), the bound version (ids, indexes, order), then the owners (`checkParams`, `checkArgs`, Delta's decoder). - **Unknown ids are ignored on both sides (Q1):** they are normal races (a `result` crossing an `abort`, a `call` @@ -1870,55 +2123,215 @@ validation, no clone of snapshots or ops. and "corrupt" as a tagged result. Limits and the threshold count bytes. No per-link compression. - **Trace carriers (R5; slice 9)** are an optional bounded field of `call` and `hold` (key count, string lengths); relays read them from the envelope. -- **Adapted bindings get diffs** (slice 7): a consumer bound through `accepts` is served like a derive of the - provider's state through the adapter: Chord keeps the last adapted value per binding, applies the adapter to the new - value and sends the diff (`diffRevisions`), so an old tab during a rollout never receives a large state whole on - every commit. +- **Adapted bindings get diffs** (slice 7b; decision "References are the only indirection" (5)): a consumer bound + through `accepts` attaches to the object instance's shared mapping for its version: per member one derive of the + instance's member through `(current) => old` (a changed member), of its whole state (a removed member), or the + member itself (unchanged), made on first use, dropped when no reference of that version is left, gone with the + instance; one map and one diff per commit per version whatever the number of old references, each hold encoding its + own ops, so an old tab never receives a large state whole on every commit. A new instance (a reload) has its own + mappings; the references move to them as to any new instance (one frame, never a new mapping applied to the old + instance's values). A version the instance does not accept gives no member: the reference waits unbound with its + last values. +- **A `failed` on a bound hold unbinds it** (slice 7b, model §5): the consumer applies it as a start-over plus the + failure (its calls waiting for a slot reject with it; the provider rejects its calls in flight once); the provider + detaches the hold's sources, drops its unsent pending ops and a queued `bound`, and keeps the ack slots of `bound` + and `state` messages already sent until their `ack` or the link's loss. A reload sends it when the object's type is + removed (`not_served`), its version is no longer accepted (`version`), or a swap parks on a failed dependency; a + restart or a parked swap still loading sends `loading`; both are followed by a fresh `bound` when the object is live + again (applied as a re-binding). A swap in place keeps the hold bound and sends it a fresh `bound` once the new + instance is live (audit A9, 2026-10-30: one rebind path; a tab's watchers may get a frame for an unchanged member). --- ## 23. Optional components -Deferred: the HTTP and channel components and middleware phases are built after the kernel fixes and a first real -proof (harness, one tool with an approval pause, a code mode reload); the playground's web shell serves its pages -with plain app code until then. The design below stays the target. +The app kit's Node components (slice 10, `slice-10.md`, decision "Slice 10's points"): `chord/next/http` and +`chord/next/workers` in `src/next-node/`; the web page host and the TUI are slice 11. Each is ordinary code on Chord's +public API; nothing in the kernel depends on them. ### HTTP (`@earendil-works/chord/next/http`) -A route is `(req, ctx) => Promise`. `req` is a plain request summary (method, path params, query, -principal, provenance such as local or relayed, selected headers; parsed and capped JSON bodies; uploads handed over in -chunks); `ctx.principal` is the request's principal (from the request resolvers), so `ctx.get(Notes, …)` gives that -principal's view. A route returns a description: +The core is one Web-standard handler built from the registry `Http` (decision "The HTTP core is a fetch handler"): ```ts -| { kind: "json"; value } | { kind: "page"; title; body } -| { kind: "bytes"; name; mime; size; read(offset, length) } -| { kind: "redirect"; to } | { kind: "error"; status } +const http = createHttp(server, { // settings are arguments, fixed for the component's life + listen: { host: "127.0.0.1", port: 8788 }, // Chord's own Node listener; none: the app mounts `fetch` + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }) }, + closeTimeout: 30_000, + onError, // HttpError +}, ctx); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }, appCore], modules }, ctx); +// an app with its own server: hono.all("/*", (c) => http.fetch(c.req.raw, { localAddress, localPort, remoteAddress })) +// plain Node: nodeRequest(message, res) converts with the listener's checks; server.on("upgrade", http.upgrade) +await http.close(ctx); // before host.close: drains running requests ``` -The component does ranges (206/416), HEAD, download headers (safe filename, `nosniff`, CSP, caching), streaming, -abort on disconnect and the CSP nonce once, instead of every plugin. Contributions hold `routes` (composed with -`keyed`; a duplicate pattern is a reported conflict, the first by plugin name wins) and `wraps`: +- **The facet** (plugin `chord.http`, passed as the same object at every reload) serves the registries `Http` + (`compose` builds the router) and `Authenticators`, and with `listen` the listener: a host-lifetime object whose + native is Chord's Node server, opened right after `load`'s publication, so a reload never closes it (decisions "The + HTTP server is a native object", "Settings are arguments, extensions are registries"). `fetch` resolves the + registries once; requests before the publication wait (503 past `readyTimeout`). +- **Contributions to `Http`:** `{ routes?, wraps?, sockets?, maxPayload?, hosts?, framers?, relays?, csp? }`, a value + or a writable (a change recomposes the router). A request uses the router as it was when it started. +- **The security baseline** (decision "The HTTP component's security baseline", on for every request and upgrade + before any plugin code): (1) **provenance**: local only when the connection's local address is loopback, `Host` is + a loopback name with the port and no `Forwarded`/`X-Forwarded-*` header; a forwarded request on loopback only + through a contributed relay whose `verify` names its origin (`{ kind: "relayed", relay, origin }`); otherwise only a + contributed host (https); duplicate or missing `Host`, userinfo in it, absolute-form targets: 403; an unparsable + target 400; `TRACE`, `TRACK`, `CONNECT` 501 (a route for them is refused at compose). A mounted handler without + `localAddress` and `localPort` is never local. (2) **Foreign requests** (Fetch Metadata) never reach the + authenticators: anonymous, except a top-level `GET` navigation to a `page: true` route. (3) A foreign `Origin` is + 403; an unsafe method from a browser needs the app's own `Origin`; a `crossSite: true` route opts out and always + runs anonymous. (4) **Default headers** on every answer, the component's own included: `no-store`, `nosniff`, + `no-referrer`, `X-Frame-Options: DENY` (without framers), a CSP (`default-src 'none'`, scripts `'self'` and a + per-response nonce `request.nonce`, `connect-src` self and its WebSocket, `frame-ancestors` the framers or + `'none'`, plus contributed `csp` sources); a response's own headers replace same-named ones. (5) `cookie(name, + value, { secret, request, sameSite?, maxAge? })` and `readCookie(request, name, secret)`: `HttpOnly`, `Path=/`, + `SameSite=Strict` or `Lax`, `Secure` on a remote origin, value and expiry signed. (6) `bytes` refuse foreign + requests, are attachments unless `inline` (then with a `sandbox` CSP), download names stripped of control and + bidirectional characters. +- **Who:** the `Authenticators` registry, in contributor order (an app needing precedence contributes one that + decides); the first principal wins, a throwing one is reported and skipped, none: anonymous. They run raced against + the request's signal and stop once it aborts. Who someone is stays app policy (D3); the registry replaces D3's + single function so plugins (a device pairing, a share) can add credentials. +- **A request's scope:** one `host.root` with its principal (an explicit `undefined`: anonymous; an anonymous scope + gets no remote object served here, `forbidden` → 403) spans matching, wraps, the route and the response until its + body is written or abandoned; its signal aborts on disconnect. `HttpRequest`: `method`, `path`, `params`, `query`, + `headers`, `origin`, `provenance`, `nonce`, `json(limit?)` (1 MiB; 400, 413), `body(limit?)`. +- **Patterns** (D8): exact segments, `:name`, a trailing `/*`; decoded segments with `/`, `\`, NUL, `.` or `..` are + 400; the most specific match wins segment by segment; a pattern from two contributions is a reported conflict, the + first by plugin name wins; `HEAD` falls back to `GET`; no route: 404. A route is a function or `{ handler, page?, + crossSite? }`. Wraps (D2) as below. + +```ts +type HttpResponse = { readonly headers?: Readonly> } & ( // D5 + | { kind: "json"; value: JsonValue; status?: number } | { kind: "html"; body: string; status?: number } + | { kind: "bytes"; mime: string; size?: number; read(offset, length, ctx): Promise; download?: string; inline?: true; ranges?: boolean } + | { kind: "redirect"; to: string; status?: 302 | 303 | 307 } | { kind: "error"; status: number; message?: string }); +``` + +`bytes` are pulled in 256 KiB reads as the consumer reads (ranges 206/416, `HEAD`, chunked without `size`, R6); no +read starts once the request's scope is sealed; a short or throwing read destroys the connection. 204, 205 and 304 +carry no body. `fileResponse(path, { download?, inline?, ranges?, mime?, root? })` stats at once (missing: 404; with +`root`, a real path outside it: 403), opens its handle at the first read, closes it when the scope seals. A throwing +route or wrap is a 500 without its message. ```ts type Wrap = { name: string; - phase: "observe" | "headers" | "transform"; // owner-defined phases, outermost first + phase: "observe" | "headers" | "transform"; // the component's phases, outermost first scope?: "own" | "all"; // default "own": only this contributor's routes route?: string; // only this route; a missing route is a reported conflict wrap: (req, next, ctx) => Promise; }; ``` -Within a phase, wraps run in contributor order (no numeric priorities: plugins have no single registration order, as -Express or Hono apps do; Fastify's lifecycle phases and encapsulation are the model). Wraps may add a security boundary -on top of an owner's checks, never instead of them. A request uses the router as it was when it started. Request -resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. +Wraps (D2) run within a phase in contributor order (no numeric priorities: plugins have no single registration +order; Fastify's lifecycle phases and encapsulation are the model). Wraps may add a security boundary on top of a +route's checks, never instead of them. + +- **Sockets** (D6): `sockets: { "/terminal/:id": (socket, request, ctx) => … }`, a `ws` WebSocket after admission + and the authenticators (no principal: 403 before plugin code), `maxPayload` per contribution (1 MiB), pinged after + 15 s of silence and terminated 15 s later. **Links:** `links.path` upgrades to `host.accept(webSocketLink(socket), { + principal, limits })`; a client that stops reading is cut off past 4 MiB waiting. **What is removed is closed at + once** (decisions Q6, "allowed hosts only through contributions"): a link or socket whose host, relay or socket + handler leaves the router is terminated and its scope sealed; an upgrade is checked again against the current router + after its authenticators answered. +- **`close(ctx)`:** new requests and upgrades refused (503), sockets and links closed (1001, terminated at + `closeTimeout`), running requests waited for up to `closeTimeout`, then sealed; before `host.close`. +- **Cost** (decision "a route's overhead accepted"): +20-21 µs per request over a plain Node handler (standard bench), + +12-13 µs warmed. `ws` is an optional peer dependency (D7). + +### Workers (`@earendil-works/chord/next/workers`) + +`createWorkers(host, Kind, { kind, entry, execArgv?, env?, authorize, principal, start, lock, idleMs?, plugins, +onOutput?, onError }, ctx)` routes `Kind` itself (`host.route`, keeping its `RouteHandle`): a forked worker per +address over IPC, the link made at the fork, a start message `{ address, start, scripts, lock }` with the app's start +data and the kind's complete current scripts (`plugins.scripts(kind)`); the 60 s start timer covers fork, lock and +connect only (Q4). `workers.state` (a JSON watchable per address key), `stop`, `retry`, `step(name)`, `close`; +`workerMain(fn, { hostPackages, redact, onError })` is the child's side (`fn({ address, start, parent, plugins, +onClose }, ctx)`, `plugins` a `WorkerPlugins` evaluating the shipped scripts, closers in reverse order). One live +worker per address (`processLock`: an abstract socket on Linux, a named pipe on Windows, a socket file elsewhere that +fails closed when stale; a start waits for a draining old holder without polling). + +- **Keep-alive** (D9, R3, Q5): the child reports when `host.activity`'s references, jobs and lingering objects and the + app's `busy` counts flip between all zero and not (local calls do not count; the first state with `ready`, which + also carries the load report); the server stops a worker idle for `idleMs` (5 min). +- **Never restarted by itself** (R2): a crash, a `fatal`, a link lost while its child lives (killed after a 5 s grace) + or a failed start leaves the address failed; its holders see `failed { waiting }` and stay on the path (Q1); + `retry(address)` or a rollout reaching workers clears it and starts the held ones (Q2). `stop` suspends the address + while it has holders (R1, Q3). +- **Errors in public and private** (decision "worker errors in public"): `state` and holders get a fixed message + (`worker failed to start`, `worker exited (SIGSEGV)`) or the child's redacted `fatal`; the full error and the stderr + tail go only to `onError` (`worker_failed`). Every app hook of a start (`start`, `principal`, `lock`, the fork) is + inside that boundary; a throwing `authorize` refuses the hold and is reported `authorize_failed`. `worker_exited` is + reported only for an exit nobody asked for. +- **Workers never build** (D11): the server's plugins component builds every plugin; workers evaluate shipped + complete sets (`createPluginRuntime`, no esbuild); `workers.step(name)` ships a rollout's set to every worker (D12: + `{ host, report } | { host, stopped: true } | { host, error }` per worker). + +### The app kit (`chord/next/app`; slice 13) + +Decisions "The app kit, point 1" to "point 5". `startServer(options)` on the server, `startWorker(options)` in a worker +entry. Opinionated and small: an app is its API, its built-in plugins and a start script. + +- **Shape:** the app's layout is fixed: `src/api/v/` (shared and per-kind modules) and the built-in plugins root + `plugins/`. The data directory is a `startServer` option (no environment variables), default a dot directory named after + the app in the home directory. One server; worker kinds are a list the app defines, each a host kind with its own + params (sharding is the app's) and how it starts (entry, start data, principal, idle time); zero kinds is a server + with clients only. +- **Principals and users:** the principal is tiny (a kind and an id) and the app's type; the kit needs only its server + and worker variants. User data lives in objects keyed by the user's id. Optional kit plugins: users (accounts with an + app-supplied profile schema), authenticators (device pairing, invite links, an ephemeral pick-a-name login). +- **Authentication:** no owner by loopback alone: the start script prints a link with a one-time token that sets a + signed owner cookie; deny by default (the HTTP component refuses requests without a principal unless the route or + socket is public); a session helper issues and revokes sessions; command-line clients send a token in a header (an + owner token file readable only by the app's OS user; a device pairing flow from other machines). +- **Storage:** one store for the server, a directory per worker under its kind and address with its store; plugins + never use data-directory paths. A files plugin per address (`put`, chunked transfers, a watchable index, caps) is + the default file storage; raw directories only where a real path is needed, owned by one object, under the worker's + directory. Export and import the data a host created (`exportData`, `importData`): the store's archive plus its + files; export stops nothing (a running worker packs its own store; without one the server reads it under the + worker's lock), import refuses an address that has data (decision "App kit naming and export"). +- **Plugins:** the app's root (required) and the user's root in the data directory; installs from git or npm; one + "apply changes" call builds and rolls out to the server, then every running worker, exposed by the app as it likes. + +### Web and TUI (slice 11) + +Decision "Slice 11's points" and "Slice 11, P3" to "P9". + +- `chord/next/web`: serving the surface, a page's client host with its connection loop (p2's `keepConnected`), loading + and applying the plugins' web facets. A base bundle per client kind built on the server from the app's entry + (content-hashed); at page load the kind's current plugin set as one combined file named by its hash, later changed + plugins one script each, each a script that registers itself (no eval); new scripts apply in place (`host.reload` in + the page); pages learn of them through an object the web component serves (the current set); a base code change + reloads the page. +- `chord/next/react`: `HostProvider`, `useHold`, `useWatch`, `useRegistry`, `useRefState`. +- `chord/next/tui`: a Node client connecting with a token, always running the downloaded code in a sandbox (decision + "The terminal's sandbox is a child process": a child process on pipes with an empty environment, `--permission` + granting nothing and no SQLite; the builtin allowlist and a frozen substitute `process` inside; a bridge terminal; + the trusted launcher outside, confirming copy, open and attach on its own prompt); a component scope (`hold`, + `watch`, `registry`, released on dispose, one `changed` callback for the app's redraw), no pi-tui dependency. + +### Messages and languages (after slice 11) + +Decision "Messages and languages in the app kit". Text people read is rendered per viewer, never by a server. + +- **Messages in code:** `defineMessages(namespace, { en, de, ... })` per plugin, in a shared `messages.ts` every + rendering facet imports; typed `t(key, ...args)`, keys and signatures from the first catalog; functions for plurals + and word order; `formatTime`, `formatNumber` through `Intl` in the viewer's locale. +- **Translations** registry (`{ namespace, locale, messages, override? }`), per host: a translation plugin contributes + its shared catalog file from each facet that renders. Fallback key by key: the locale's base, then the first catalog + (reported once); conflicts: the first by plugin name wins. Catalogs never cross links. +- **The locale** is a per-principal preference (device default, live switch, `lang` and `dir` on the page). +- **Messages as data:** `t.message(key, ...args)` gives `{ namespace, key, params }` for state, notices and errors; + each client renders it with `t.render` (checked at runtime across a link). Prompts, model output, logs and error + codes are never translated. ### Channels -WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links, with their own flow -control (acknowledged windows). +WebSocket byte channels with acknowledged windows, separate from object links, wait for a second user after the +terminals (slice 10 D6); they would be built on top of plugin sockets. ### Chunked transfers @@ -1961,14 +2374,20 @@ loop. Chord has no stream type (it would need its own rules for drops, cancellat loading object (refines R17). A missing tool gives its call an error result the model sees; a missing model makes the generation report "model unavailable" and wait (not fail) until a model contribution appears; a task whose definition is missing stays blocked (`missing_task`) until the plugin is installed or the task is aborted, as Durable does. +- **Durable's views from other objects** (slice 8 §5): the harness object contributes its `Native` to a + registry of p3's; another object declares that registry (`self.registry`), wakes on it (`self.retryOn`) and follows + a view through its value (`self.follow`). A Session close ends Durable's synchronous observers `failed` ("session + closed"; decision "Slice 8 S1"; app watches still end `ended`), so the follower fails `transient` and follows the + next Harness once its contribution arrives. `registryReader`'s listener runs inside Chord's batch: it only wakes + Durable's scheduler (a Chord call there throws `invalid/reentrant`). - `HarnessModels` and `HarnessEnvironments` start with one contribution each, from the models and envs plugins' objects that own the natives; several providers or environment kinds are their `compose`. After a models reload the next request uses the new runtime; a running stream finishes on the old one. - **Chord from Durable code (Q6):** tools, hooks and sections get Durable's `ToolExecutionApi` and a `Context`, never a Chord `ctx`. p3's core opens the harness with a base context holding the host under a typed key (`withContextValue(ChordHostKey, host, …)`); every phase context inherits it. A core helper opens a call scope from - any Durable context: `chord(async (ctx) => …, context)` = `host.root(fn, { signal: context.abortSignal, telemetry: - context.telemetry })` (host principal; attribution as data; references released when `fn` settles). No wrapping of + any Durable context: `chord(async (ctx) => …, context)` = `host.root(fn, context)` (host principal; attribution as + data; references released when `fn` settles; slice 8 D3). No wrapping of tools and no p3 `defineTool`. Clients are never reachable this way (they are not addressable): a tool that needs the user goes through an owner's state, e.g. the Questions service (`apps/p2/docs/questions.md`): the model-facing `ask_user` tool and any other tool call `Questions.ask`/`wait`, tabs render the open questions and `answer` them. @@ -2033,8 +2452,7 @@ this document): | 11 | where a flow lives | 20 | when login is ported | | 14 | project plugins | 21 | own pass over `plugin-loading.md` | | 16 | a Durable Object profile | 24 | when a DO target becomes concrete | -| 24 | the rollout install timeout (60 s) and a per-type drain override: host options or fixed (the routine handover uses `closeTimeout`; a failing job `finish` backs off 1-60 s and blocks after five failures, decided) | 3 | with slice 7 | -| 26 | how a migration commit reaches consumers (`[["r", value]]` per changed member, or a fresh `bound`; adapted by `accepts` for older consumers) | 21, 22 | slice 7 | +| 24 | a per-type drain override: a host option or fixed (no install watchdog: `reload` takes the app's context, decision "Slice 7b's open points" (4); the routine handover uses `closeTimeout`; a failing job `finish` backs off 1-60 s and blocks after five failures, decided) | 3 | if a real case needs it | | 28 | Astra's circuit breaker (a hard transport-buffer cap plus a stall deadline); the heartbeat is built, the cap was never decided | 3 | if a real case needs it | | 29 | round-3 cut C11 (`ctx.trace`); C3 and C4 are decided (§14, §15) | 8 | slice 9 | @@ -2080,7 +2498,8 @@ const accepted = checkAccepts(NotesV2, { - A history starts with a base written in full with an explicit version (`remote()` / `local()`; `local` has flat `members` and no variants). `evolve(prev, (e) => …)` always continues it with structured edits: `state`, `addState`, `removeState`, `args`, `result`, `method(name, args, result)`, `addMethod`, `removeMethod` (local: no - variant argument). Every version is materialized and fingerprinted at declaration, so an old plugin's own copy of + variant argument), and for a local contract's keyed members `keyed`, `addKeyed`, `removeKeyed` (slice 7b; owner, + 2026-10-26: keyed members' versions are mapped per key). Every version is materialized and fingerprinted at declaration, so an old plugin's own copy of v1 computes the same fingerprint; no build step, no generated files, no hand-copied fingerprints. - **A new history base** (a params change or a change of the variant set) is a new `remote()`/`local()` with an explicit version and no accepts for older versions: old plugins get `contract_mismatch` and must update. @@ -2088,14 +2507,15 @@ const accepted = checkAccepts(NotesV2, { the objects layer) requires, at type level and again at runtime, a mapping for every member that exists in the old version and changed or was removed since: changed state `(currentMember) => oldMember`, removed state `(current variant state) => oldMember` or `"refuse"`, `args: (old) => current`, `result: (current) => old`, removed methods - `{ call }` or `"refuse"`; added members need nothing. No-op edits need identity mappings. Untyped JavaScript + `{ call }` or `"refuse"`, changed keyed members `(current document) => old document` or `"refuse"` (mapped per key + for an older local consumer), removed keyed members `"refuse"`; added members need nothing. No-op edits need identity mappings. Untyped JavaScript providers fail at declaration naming the member; unknown keys and wrong shapes are rejected. - **The fingerprint** is a hash (`hash64`, portable, not a security boundary) of a schema-aware, data-only normal form (`NORMAL_FORM_VERSION`, tied to the protocol): annotations dropped only on schema nodes, `required`/`enum` sorted, - union and intersection order kept (it is meaning: defaults), a whitelist of TypeBox keywords, executable features + union and intersection order kept (it is meaning: defaults), a whitelist of JSON Schema keywords, executable features rejected; bytes are a `{ type: "bytes" }` data marker. Schemas are deep-copied into a private history (never frozen; immutable by convention). - **Schema helpers:** `str num int bool nul lit oneOf union obj arr map opt bytes unknown` and `type Static`. -- Chord wraps state mappings as a derive and diffs per binding (section 22); members an old consumer does not know do +- Chord wraps state mappings as a derive and diffs per object and version (section 22); members an old consumer does not know do not appear. --- @@ -2118,8 +2538,8 @@ function doing counted work counts it through its context; call sites never coun site labels. **Testing the core:** every slice spec contains an explicit test list, one group per mechanism with its edge cases, -tested individually and exhaustively: activation and dependency recovery (R18: broken dependency, trigger, backoff as -safety net, a dependency failing while live, cycles), revocation over running jobs (R11), compatibility (R6), params +tested individually and exhaustively: activation and parked dependencies (the dependency model: unbound requests, waits without timers, wakes on every +change, retry passing on, a dependency failing while live, cycles, the start barrier's first outcomes), revocation over running jobs (R11), compatibility (R6), params and arguments for every caller (R7), migration results and old calls on migrated documents (R8, R15), flow control (the window per origin, folding, relays forwarding and returning acks), the incarnation fence (R20), the harness start after the barrier (R17), native restart patterns (R16), and W1's publication list (a throwing derive, a chain A→B→C @@ -2193,7 +2613,7 @@ before/after per hop: `apps/p2/docs/p3-reviews/v2-cost-model.md`. | heartbeat | one timestamp write per received message; one timer per host | | apply | `applyImmutable`, structure shared | | new hold | the value encoded once (unavoidable) | -| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | +| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted member, object instance and version (while a reference of the version holds it) | **Cost guards:** Chord reports full-value walks, deep copies, encodes, schema validations and materializations as ordinary telemetry metrics, always on (no test mode; with the NOOP adapter they cost an empty call where such work @@ -2215,9 +2635,8 @@ with their contributor, readable contributions and readers looking up the curren registry keys with a `conflict` flag; `restart_required`; detached references counting toward presence; contracts taken from the running app; ceremony (`native/` directories, `defineNative`, lints); weakening type checking; "magic" input-field matching for delete; a -factory trial; a migrate dry run; principal field patterns and re-stamping for revocation; `storeKey`; a consumer contract check during staging; reload votes; per-plugin rollback within a -host; `defaultProjectTrust: "always"`; loader variants A and B; a topology layer above the host API; a stream type in -Chord; callbacks as method arguments; `opaque` members; numeric middleware priorities; placement as data naming an +factory trial; a migrate dry run; principal field patterns and re-stamping for revocation; `storeKey`; a consumer contract check during staging; reload votes; `defaultProjectTrust: "always"`; loader variants A and B; a topology layer above the host API; a stream type in +Chord; callbacks as method arguments; `opaque` members; numeric middleware priorities; host addresses as data naming an input field; host-name strings in app code; host values (start data and platform bindings are core objects); consumer-side linger; registry composition as a staging check; a JSONL document storage; `tx.get` for documents; validating state and documents on commits or reads (migration results are validated once, R8); copying values on same-host calls; op-rewriting projections and a view algebra; keeping natives across reloads by name; Chord ending the process on a storage failure; an `optional` diff --git a/packages/chord/docs/next/explainer/book.md b/packages/chord/docs/next/explainer/book.md deleted file mode 100644 index 05a98d9aa..000000000 --- a/packages/chord/docs/next/explainer/book.md +++ /dev/null @@ -1,4507 +0,0 @@ ---- -title: Chord v2: a guide to building apps that reload -cover: Chord v2 / A Guide -kicker: A guide · Chord v2 target design · 2026-10-20 -subtitle: How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running -stats: - - 11 | layers in the rebuild, L1 to L11 - - ~1,300 | lines built so far (L1 contracts) - - 6–7k | target size of Chord v2, without tests - - 0.004 ms | Delta's apply per small change -note: | - This guide teaches the **designed** API of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) - from `packages/chord/docs/next/design.md` and the newer decision log `packages/chord/docs/next/decisions.md`, which wins - where the two differ. Most of it is **not built yet**: a first kernel of about 10k lines is frozen as a reference and - Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (`src/next/contracts/`, commit - `7d2bb8ea8`). Code in this guide is therefore the target, not something you can run today. Every chapter says what is - built. API names are working names. Notes marked *review* are points that are undecided, or where the sources - disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start. -colophon: Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README ---- - -# Before we begin {.preface} - -It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session -S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through `bash`. A colleague, -Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way -round. He fixes one line in the notes plugin on the laptop and presses **Reload**. - -What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The -agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is -lost, and no process restarts. - -The obvious ways to get there fail. **Restarting the processes** loses the turn in flight, the open terminal and every -connection; pi-durable would resume the turn, but the test run and the connections are gone. **Hot-swapping modules** -in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that -nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the -old code is never closed, and two versions of the plugin now write the same data without any order. - -Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: **every piece -of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only -declare what they need.** Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, -watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only -what touches the operating system, and route calls between browser tabs, the server and per-session worker processes -without the plugin author writing any of that. - -This guide teaches Chord v2 by building an app called **Studio**: a small version of p2 (Pi Mobile), with a server, -one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first -chapters grow one plugin, `@studio/notes`, until it uses every concept; the last three chapters assemble the rest of a -p2-like app from the same parts. - -:::review | Review: what exists today -Nothing in this guide runs yet except the contract layer. `packages/chord/src/next/contracts/` (L1 part 1: `remote`, -`local`, `method`, `evolve`, `checkAccepts`, the schema helpers) is built and reviewed. The rest of L1 (inputs, -argument checks, principals, errors) is a draft spec (`packages/chord/docs/next/layer-1.md`). The frozen kernel in -`src/next/*.ts` implements an older version of the design (host values, a development freeze, JSONL storage, -descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code. -::: - -## Ten things to know - -:::findings -1. **An app is a tree of hosts.** One server, any number of workers it starts on demand, any number of clients (tabs, - TUIs) that connect to it. Clients reach workers only through the server, which relays. -2. **Objects are the unit of everything.** An object type is a contract (name, input, placement, members, version); an - object is one instance per input and caller. The first `get` creates it, later gets share it, the last release ends - it. -3. **Factories declare, Chord activates.** A factory is synchronous: it declares documents, references, derived - values, OS handles, routines and jobs through `self`, and returns the members. Chord opens, binds and starts them, - and undoes them in reverse. -4. **Links stamp who is calling.** A caller's data never says who it is; the link it arrived on does. Each caller gets - its own object and the variant of members it may use. -5. **Shared data lives once, behind views.** A `local` shared store holds the data; per-caller `remote` views expose - what each caller may see. Only stable identities persist documents. -6. **State changes only in transactions.** Documents are JSON values written in `self.tx`; commits are atomic for - every viewer and travel as small operations, never as whole values. -7. **Failures look like disconnects.** A reference survives link drops, restarts and failures: calls park or fail, - values stay readable, and Chord retries by itself. There are no request ids. -8. **Everything long-lived has a declared form.** Standing behaviour is a routine, work that outlives a call is a job, - input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer - or a callback. -9. **Reload swaps pure objects in place and restarts native ones.** Objects without OS handles get the new code with - their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole - host on its old code. -10. **pi-durable runs inside a core object.** The agent harness is a host-lifetime object on each worker; models, - environments and tools reach it through registries, and tools reach Chord through `chord(fn, context)`. -::: - -## Words used throughout - -A **host** is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical -identity and an **incarnation** that changes on every start. A **link** connects two hosts. The **server** is the one -addressable root host; a **worker** is a host the server starts for one session; a **client** is a tab or TUI, not -addressable. A **plugin** is a directory with a `package.json`; its code for one kind of host is a **facet**. **Base -code** is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while -plugins reload. An **object type** (or **contract**) describes objects; an **object** is one instance; its **input** is -its key among objects of that type, and its **placement** computes from the input on which host it lives. A **factory** builds an -object; **`self`** is what a factory declares through. A **principal** is who calls, stamped by the link; a -**variant** is the set of members a principal gets. A **reference** is what `get` returns. A **readable** is a value -plus its ordered changes. A **document** is a named JSON value an object owns, written in a **transaction**. A -**derive** is a value computed from readables. A **native** is an OS-level value (a socket, a watcher, a process) -opened with `self.open`; an object with natives is a **native object**, one without is a **pure object**. A -**routine** is standing behaviour, a **job** is owned work that outlives its call, a **flow** is input into running -work. A **registry** is an owner's extension point, filled with **contributions**. **`ctx`** is what every call -receives: cancellation, caller, call-scoped references. A **reload** swaps a plugin's code inside running hosts; a -**restart** ends a process; a **revision** names the complete code set the hosts run. - -:::note | Reading the code in this guide -Imports are shown once per file and then omitted. `obj`, `str`, `arr` and friends are Chord's schema helpers (they -return ordinary TypeBox schemas). Helper functions that belong to the app (`authenticate`, `watchDir`, `uuidv7`) are -named and not shown. Where the design gives no name, the guide says it uses a working name. -::: - -# The Shape of an App {#shape lede="Which processes a Chord app has, how they find each other, and the start code that connects them."} - -Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, -and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does -none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare -deployment routes through the edge. This chapter writes Studio's start code: `server.ts`, `worker.ts`, the browser -entry, and a test setup. - -## Three roles - -A Node app on Chord has one shape: - -:::flow wide | The three roles. Clients connect to the one server; the server starts workers and relays between them. -row "Clients: not addressable" muted -box tab1 "browser tab" "owner, local" -box tab2 "browser tab" "guest Ann" -box tui "TUI" "owner" -row "Server: exactly one" accent -gap -box server "server" "`Server()`" accent -gap -row "Workers: one per session, on demand" blue -box w1 "worker" "`Session({ id: \"S1\" })`" blue -box w2 "worker" "`Session({ id: \"S2\" })`" blue -box w3 "worker" "`Session({ id: \"S3\" })`" dashed -arrow tab1 -> server "WebSocket" -arrow tab2 -> server "WebSocket" -arrow tui -> server "WebSocket" -arrow server -> w1 "IPC" -arrow server -> w2 "IPC" -note "A worker for S3 does not exist until something asks for an object that lives there." -::: - -| Role | Addressable | Who it is comes from | Lifecycle | -| --- | --- | --- | --- | -| client | no | the authentication of its link | connects by itself | -| server | yes, exactly one | its own declaration | runs | -| worker | yes, many, by a typed address | stamped by the server, its parent | started on demand by the app | - -The **server** holds what all sessions share (the session list, access, settings) and starts workers. A **worker** -runs one session: its agent, its files, its terminal connections. **Clients** are browser tabs and TUIs. Nobody can -send a request *to* a client, so a client can only serve objects that nobody outside it needs; chapter IV names them -`local` types. - -Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a -remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. -A detached process such as a terminal host could become another kind of child of the server. - -## Host kinds and addresses - -Every addressable host has a typed address. The app declares its **host kinds** once, in base code: - -```ts -// studio/src/hosts.ts -import { hostKind, obj, str } from "@earendil-works/chord/next"; - -export const Server = hostKind("server"); -export const Session = hostKind("session", obj({ id: str() })); - -Server(); // the address of the server -Session({ id: "S1" }); // the address of the worker of session S1 -``` - -There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the -kinds. An object type names its host with a placement function from its input to an address (chapter II), so the -type checker knows that `Session({ id: session })` needs a `session` field. - -## Links - -A **link** is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory -pair for tests, IPC for a Node child process forked with `serialization: "advanced"`, and WebSocket. All three carry -the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: -IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep. - -Each link has a **direction**. A host has at most one link to its parent (`connect`) and any number of links from -below (`accept` for clients, `route` for workers). Requests arriving on a link act as the principal the link was given; -chapter III explains why that matters. - -## The server - -```ts -// studio/src/server.ts -import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next"; -import { fork } from "node:child_process"; -import { WebSocketServer } from "ws"; -import { Server, Session } from "./hosts.ts"; -import { coreServerFacet } from "./core/server.ts"; -import type { P } from "./principal.ts"; - -const server = createHost

({ - address: Server(), - principal: { kind: "server" }, // who this host's own code acts as - storage: await openServerStorage(dataDir), // persisted documents (chapter V); memory if omitted - onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); }, -}); - -await server.load([ - { name: "@studio/core", facet: coreServerFacet(config) }, // base code, under a fixed name - ...(await loadPlugins("server")), // [{ name, facet }] from the plugin loader -]); - -const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 }); -wss.on("connection", (ws, req) => { - const principal = authenticate(req); // app code: cookie, loopback, device token - if (!principal) return ws.close(4401); - server.accept(webSocketLink(ws), { principal }); -}); - -server.route(Session, { - authorize: (p, { id }) => - p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id), - resolve: async ({ id }, { signal }) => { - const principal: P = { kind: "worker", session: id }; - const child = fork("worker.ts", [], { serialization: "advanced", signal }); - child.send({ id, principal, init: await initFor(id, signal) }); // the app's own start message - return { link: ipcLink(child), principal }; // Chord stamps `principal` on it - }, -}); -``` - -Each call has one job: - -| Call | Meaning | -| --- | --- | -| `createHost

({ address?, principal?, ...options })` | with an address: an addressable host whose own code acts as `principal`; without: a client. `P` is the app's principal type | -| `load([{ name, facet }])` | the facets this host runs, each with its plugin's `package.json` name; duplicates and invalid names are refused. Base code passes its core facet the same way | -| `accept(link, { principal })` | a link from a client; every request on it acts as `principal` | -| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind. `authorize(principal, address)` runs before anything starts; `resolve(address, { signal })` returns `{ link, principal }` | -| `connect(link, { principal })` | the link to the parent, at most one, replaced after a reconnect; `principal` says who the parent is | -| `root(fn, { principal?, signal? })` | a call scope for code outside objects (a CLI command): gives `fn` a `ctx` (chapter VI) | - -Chord has no `listen`. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay -delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls -`accept`. Radius, for example, is one more line: - -```ts -radius.onChannel((ws) => - server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); -``` - -## The worker - -```ts -// studio/src/worker.ts -const { id, principal, init } = await startMessage(process); // app code: the first IPC message -const worker = createHost

({ - address: Session({ id }), - principal, // { kind: "worker", session: id } - storage: await openSessionStorage(init.dataDir, id), // one storage per session -}); -await worker.load([ - { name: "@studio/core", facet: coreWorkerFacet(init) }, // serves the start data (chapter IX) - ...(await loadPlugins("worker")), -]); -worker.connect(ipcLink(process), { principal: { kind: "server" } }); -``` - -## A browser tab - -```ts -// studio/web/main.tsx -const client = createHost

(); // no address: a client -await client.load(await loadPlugins("web")); -client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }); -const notes = await client.get(Notes, { session: "S1" }); // tab → server → worker S1 -``` - -## The first `get`, step by step - -The tab asks for the notes of S1. No worker for S1 runs yet. - -:::sequence animate | The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link. -actor tab "browser tab" "owner" -actor server "server" accent -actor app "route(Session)" "app code" muted -actor worker "worker S1" blue -tab -> tab: placement: Session({ id: "S1" }) -tab -> server: hold Notes { session: "S1" } -server -> server: not mine: route(Session) -server -> app: authorize(owner, { id: "S1" }) -app --> server: true -server => app: resolve({ id: "S1" }) -app -> worker: fork + start message -worker -> worker: createHost, load, connect -app --> server: { link, principal: worker S1 } -server => worker: hold (stamped: owner) -worker -> worker: placement check, run factory -worker --> server: bound: snapshot -server --> tab: bound: snapshot -::: - -`resolve` runs once per address while its link lives. When the link drops (the worker crashed or the supervisor -stopped it) and references to that worker still exist, Chord calls `resolve` again by itself, with a per-address -backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes -even if the request that triggered it went away. - -**Incarnations** make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link -handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold -(chapter VI). - -## Tests: everything in one process - -The same code runs in a test with memory links. `resolve` builds the worker host in memory and returns one end of the -pair: - -```ts -server.route(Session, { - authorize: () => true, - resolve: async ({ id }) => { - const [up, down] = memoryLinkPair(); - const principal: P = { kind: "worker", session: id }; - const worker = createHost

({ address: Session({ id }), principal }); - await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]); - worker.connect(up, { principal: { kind: "server" } }); - return { link: down, principal }; - }, -}); -``` - -Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path. - -## What stays app code - -Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord -provides one readable for the last one: `host.activity`, with `{ references, calls, jobs, lingering }` (viewers, calls -in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server -together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for -five minutes, as p2 does. Chord counts; it has no idle policy. - -## Host options and timeouts - -`createHost` takes the options introduced later in this guide: `storage` (V), `principalKey` and `revokedBy` (III, -VII), `telemetry` (XVII) and `onError`. `onError` receives background errors and `{ kind: "storage_failed" }`; every -app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor -restarts it. - -Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: -`readyTimeout` (60 s: how long an activation and an awaited `get` wait) and the park deadline (60 s: how long a call -waits through a link drop or a restart before failing `unavailable`). - -:::note | Built so far -Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has -working versions with the same shape (`createHost`, `connect`, `accept`, `route`, `load`), which the rebuild -simplifies. -::: - -# A First Object {#first-object lede="A plugin, a contract, a factory, and a browser tab that calls it on another process."} - -Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can -add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent -tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord -there is **one object** for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it. - -## The plugin's files - -A plugin is a directory with a `package.json`. Its `name` (npm style, globally unique and stable) is the plugin's -identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a -directory name or load order. - -| File | Runs on | Holds | -| --- | --- | --- | -| `package.json` | read by the loader | `"name": "@studio/notes"` | -| `contract.ts` | every host that imports it | object types: names, shapes, method signatures; no running code | -| `worker.ts` | every session worker | the worker facet | -| `server.ts` | the server | the server facet (later: the download route) | -| `web.tsx` | browser tabs | the web facet: the panel | - -## The contract - -An object type is a contract: a name, an **input** (the object's key among objects of this type), a **placement** -(on which host it lives, computed from the input), the **members** (state you can read and methods you can call) and a **version**. - -```ts -// notes/contract.ts -import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next"; -import { Session } from "@studio/core/hosts"; - -export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() }); -export type Note = Static; - -export const Notes = remote("notes", { - version: 1, - input: obj({ session: str() }), // the key: { session: "S1" } - placement: ({ session }) => Session({ id: session }), // input → the host's address - variants: { - editor: { - state: { notes: obj({ items: arr(Note) }) }, - methods: { - add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), // returns the new id - remove: method(obj({ id: str() })), // no result - }, - }, - }, -}); -``` - -`remote` means objects of this type can be reached from any host, across links (chapter IV introduces `local`). The -schema helpers (`obj`, `str`, `int`, `arr`, `map`, `opt`, `lit`, `oneOf`, `union`, `bool`, `num`, `nul`, `bytes`, -`unknown`) return ordinary TypeBox schemas; short options such as `{ min, max }` bound lengths or values. Schemas must -describe data: no callbacks, no class instances, no custom codecs. - -`method()` has four forms: `method()` (no arguments, no result), `method(args)`, `method(args, result)` and -`method(undefined, result)`. - -A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its -description and computes a **fingerprint**, a hash of its input, variants and members. Two hosts compare contracts by -version and fingerprint (chapter XVI). Placement is not part of the fingerprint. - -## Placement - -The input says *which* object; `placement` says *where* it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server -routes by it. The provider runs the function too and refuses an object that is not its own with `invalid/placement`. -So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed -function, a `Session` placement on a type whose input has no `session` field does not compile. - -Inputs are **normalized** before anything else: defaults are applied, unknown fields removed, and a canonical key -computed. `{ session: "S1" }` and `{ session: "S1", extra: 1 }` are the same object. - -## The facet and the factory - -A **facet** is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the -host which object types it serves: - -```ts -// notes/worker.ts -import { defineState } from "@earendil-works/chord/next"; -import { defineFacet } from "@studio/core"; // Studio's defineFacet: principals typed as Studio's P (chapter III) -import { Note, Notes } from "./contract.ts"; - -const NoteList = defineState("notes.list", { // a document (chapter V); in memory for now - schema: obj({ items: arr(Note) }), - version: 1, - initial: () => ({ items: [] }), - checkpointWhen: () => true, -}); - -export default defineFacet((host, self) => { - host.serve(Notes, ({ session }, principal, self) => { - const list = self.state(NoteList); // THE note list of this object - return { - variant: "editor", - notes: list, // a document handle is a readable: expose it as state - add: ({ text }, ctx) => - self.tx(async (tx) => { - const id = uuidv7(); - (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() }); - return id; - }, ctx), - remove: ({ id }, ctx) => - list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), - }; - }); -}); -``` - -Read `host.serve(Notes, factory)` as "this host builds `Notes` objects like this". The factory receives the -normalized input, the caller's principal (chapter III) and `self`, through which it declares what the object needs. -It returns the members of one variant, tagged with `variant`. - -**Factories declare; Chord activates.** The factory is synchronous and has no side effects. It does not await, open -files or start timers. `self.state(NoteList)` does not load anything: it declares that the object has this document -and returns a **handle** that stays valid for the object's whole life. Chord then performs the activation: it loads the -document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for -reload: a reload runs the factory again, and anything the factory did itself would happen twice. - -## Calling it - -```tsx -// notes/web.tsx -export function NotesPanel({ session }: { session: string }) { - const notes = useRef(Notes, { session }); // app-side hook over host.get / release - const items = useReadable(notes?.notes)?.items ?? []; - return ( -

- {items.map((n) =>

{n.text}

)} - notes?.add({ text })} /> -
- ); -} -``` - -Underneath the hooks is the reference API: - -```ts -const notes = await client.get(Notes, { session: "S1" }); // live and hydrated once awaited -notes.notes.value.items; // the current list -notes.notes.subscribe((v) => render(v.items)); // every committed change -const id = await notes.add({ text: "check the flaky test" }); -notes.release(); // the object may end now -``` - -An awaited reference is **live and hydrated**: every state member has a value. State members are **readables**: -`.value` is the current value, `subscribe` is called after every commit. The first `get` for `{ session: "S1" }` runs -the factory; every later `get` for the same input and caller shares that object. The object ends when the last -reference is released (a type may keep it a little longer with `linger`). - -:::sequence animate | A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change. -actor tab "browser tab" -actor server "server" muted -actor worker "worker S1" blue -actor obj "Notes { S1 }" accent -tab -> server: call add { text } -server -> worker: relay (header only) -worker -> obj: strip + validate args -obj => obj: self.tx: push note, commit -obj --> worker: result: id -worker --> server: state: ops of the commit -worker --> server: result -server --> tab: state, then result -::: - -## Two rules for members - -**Methods return data, never objects.** A method that wants to point you to another object returns its address -(`{ session: "S2" }`), and you `get` it yourself. That keeps every reference owned by someone who releases it. - -**No callbacks as arguments.** A callback carries code, and everything it closes over, from one object into another. -Suppose an inbox object passed `onFile` to another object, which stored it. After a reload of the inbox's plugin, the -other object still calls the *old* `onFile`; after the inbox ends, it calls into an object whose file watcher is -already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes -through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII). - -## What is wrong with this version - -This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the -`editor` variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as -chapter III shows, a remote object exists per caller, so each caller's `Notes` object has its own `NoteList`. - -:::note | Built so far -`remote`, `method`, the schema helpers, `Static` and fingerprints are built (`src/next/contracts/`). Serving, `get`, -references and documents are layers L3 to L5. -::: - -# Who Is Calling {#principals lede="Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members."} - -The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote -what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In -Chord the answer is a **principal**, a small JSON value the app defines and the links stamp. - -## The app's principal type - -Chord defines no principal kinds; the app does. Every variant has a `kind`. Studio uses p2's set: - -```ts -// studio/src/principal.ts -export type P = - | { kind: "owner"; access: "local" | "relay" | "device"; device?: string } - | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number } - | { kind: "server" } - | { kind: "worker"; session: string }; -``` - -| `kind` | Who | -| --- | --- | -| `owner` | the person running the app, on loopback, through the relay, or on a paired device | -| `guest` | someone invited into one session, with a mode and the grant (and its revision) that let them in | -| `server` | the server's own code calling a worker | -| `worker` | a session worker calling the server | - -## Links stamp principals - -A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, -and stamps the result on the whole link (`server.accept(link, { principal })`). The server tells Chord who each worker -is when it resolves it (`return { link, principal }`). When the server relays a client's request to a worker, Chord -writes the client's principal into the forwarded request; a principal field coming from a client is always -overwritten, and only a trusted parent link may carry one. - -Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest -who opens it gets a guest record (grant `g1`) and a cookie; `authenticate(req)` turns the cookie into -`{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 }` on every new connection. Chord never sees -cookies or grants, only the finished principal. - -## The caller is part of the identity - -A `remote` object exists once per **(type, input, principal key)**. The **principal key** is a string computed from -the principal; by default its canonical JSON (sorted keys). The app may pass its own `principalKey` to `createHost`, -and must pass the same function on every host, or identities split silently. - -```ts -const principalKey = (p: P) => - p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}` - : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}` - : p.kind === "worker" ? `worker:${p.session}` - : p.kind; -``` - -The design wants the key to include how the client is connected and the grant's revision, so that two access paths -never share objects and a changed grant never reuses the old grant's object: - -| Caller | Principal key | `Notes { S1 }` object | -| --- | --- | --- | -| owner, tab A, local | `owner:local:` | 1 | -| owner, tab B, local | `owner:local:` | 1 (shared with tab A) | -| owner through the relay | `owner:relay:` | 2 | -| guest g1, revision 1 | `guest:g1:r1` | 3 | -| guest g1 after a change to write mode, revision 2 | `guest:g1:r2` | 4 (new) | - -## Variants - -The contract lists named sets of members; the factory receives the principal and returns the variant that principal -may use: - -```ts -// notes/contract.ts (version 1, final form for now) -variants: { - editor: { - state: { notes: obj({ items: arr(Note) }) }, - methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) }, - }, - reader: { - state: { notes: obj({ items: arr(PublicNote) }) }, // no `by`: authors are hidden - }, -}, -``` - -```ts -// notes/worker.ts -host.serve(Notes, ({ session }, principal, self) => { - if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … }; - return { variant: "reader", notes: … }; -}); -``` - -A reader has no `add` at all. Chord refuses a call to a member that is not in the caller's variant, so `add` needs no -check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every -variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per -variant (the reader's notes have no authors). `remote(...)` enforces them at the type level, and at runtime for -contracts that arrive as data. - -## How a facet knows `P` - -`P` is Studio's type, not Chord's, and every host is created with it (`createHost

`). A plugin's facets are written -and compiled separately, then handed to `host.load` as values, so the type checker cannot infer `P` from the host into -the factory's `principal`. Studio's base code therefore exports its own `defineFacet`, already bound to `P`, and every -facet of every plugin imports that one instead of Chord's: - -```ts -// studio/src/facet.ts (base code, exported as @studio/core) -import { bindFacet } from "@earendil-works/chord/next"; -import type { P } from "./principal.ts"; - -export const defineFacet = bindFacet

(); // a facet's principals are Studio's P -export type { P, Revocation } from "./principal.ts"; -``` - -```ts -// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet) -import { defineFacet } from "@studio/core"; - -export default defineFacet((host, self) => { - host.serve(Notes, ({ session }, principal, self) => { // principal: P - if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … }; - … - }); -}); -``` - -The import is cheap: `@studio/core` is a host package plugins already import at runtime (the contract imports `Session` -from it), `P` is a type-only import, and `bindFacet` is a one-line function. The host checks the pairing: `load` -accepts only facets typed for its own `P`, so a facet compiled against another app's principal type, or one built with -Chord's untyped `defineFacet`, does not compile into Studio's start code. Client facets (`web.tsx`, `tui.ts`) use the -same `defineFacet`; they rarely see a principal, because clients serve only `local` types. - -None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link -stamped. - -:::review | Review: binding facets to the app's principal type (open) -The design shows `defineFacet((host, self) => …)` without saying how `P` reaches the factory. The frozen kernel takes -it as an explicit type parameter, `defineFacet

(…)`, defaulting to `{ kind: string }` -(`packages/chord/src/next/facet.ts`), and the playground writes `defineFacet(…)`. The pre-bound -`defineFacet` above (`bindFacet` is a working name) is the owner's preferred direction, recorded as an open point in -`packages/chord/docs/next/layer-1.md` §4. -::: - -## Narrowing on the caller's side - -Owner and guest run the same `web.tsx`, and which variant they get is known only at runtime. The types force every -caller to narrow before using variant members: - -```ts -const notes = await client.get(Notes, { session: "S1" }); -notes.add({ text: "hi" }); // type error: `add` exists only in "editor" -if (inVariant(notes, "editor")) await notes.add({ text: "hi" }); -expectVariant(notes, "editor"); // or assert (throws) -function Editor({ notes }: { notes: RefIn }) { … } -``` - -The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the -types gets the same error. - -## Admission before a worker starts - -Principals also decide whether a caller may reach a host at all. `route(Session, { authorize })` runs before Chord -resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for -them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate -this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can -see and do is decided by the variant its factory picks. - -## Inside methods - -`ctx.principal` is the caller of the current call. In Studio's notes the editor's `add` uses it to record the -author: - -```ts -add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx), -``` - -Where principals act, in summary: - -| Purpose | Where | -| --- | --- | -| identity: which object a caller gets | principal key | -| variant: which members | the factory | -| admission: whether a host is reachable | `route.authorize` | -| attribution and fine checks | `ctx.principal` in methods | -| revocation | `revokedBy` (chapter VII) | - -## The bug from chapter II - -Now the bug is visible. The owner's `Notes { S1 }` and Ann's `Notes { S1 }` are two objects (rows 1 and 3 above), and -each declared its own `NoteList`. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; -they must be views over something that exists once. That is the next chapter. - -# One Store, Many Views {#shared-store lede="Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it."} - -Every caller has its own `Notes` object, which is what makes per-caller variants possible. But data must exist once. -The fix is the most common pattern in Chord: a **shared store**, a `local` object that holds the data and does the -heavy work once, behind thin per-principal **views**, the `remote` objects callers get. - -## `local` types - -```ts -// notes/contract.ts -export const NoteStore = local("notes.store", { - version: 1, - input: obj({ session: str() }), - members: { - notes: obj({ items: arr(Note) }), // state: a schema - public: obj({ items: arr(PublicNote) }), - add: method(obj({ text: str(), by: str() }), str()), // methods: method(...) - remove: method(obj({ id: str() })), - }, -}); -``` - -A `local` contract has flat `members` and no variants. The differences: - -| | `remote` | `local` | -| --- | --- | --- | -| reachable from | any host, across links | its own host only | -| identity | (type, input, principal key) | (type, input) | -| principals, variants | yes | no | -| factory | `(input, principal, self)` | `(input, self)` | -| arguments | stripped to the provider's schema and validated, for every caller | see the review note below | -| values on the same host | passed by reference, never copied | passed by reference, never copied | -| `ctx.principal` | the caller | none | -| may persist documents | no (chapter V) | yes | - -A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote -this note") is passed as data (`by`), by the view that knows who called. Clients serve only local types, because -nobody can address a client. - -## The pattern - -```ts -// notes/worker.ts -export default defineFacet((host, self) => { - host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); - const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); - return { - notes: list, - public: pub, // computed once, for every reader - add: ({ text, by }, ctx) => - self.tx(async (tx) => { - const id = uuidv7(); - (await tx.draft(list)).items.push({ id, text, by, at: Date.now() }); - return id; - }, ctx), - remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), - }; - }); - - host.serve(Notes, ({ session }, principal, self) => { - const store = self.get(NoteStore, { session }); // a reference, declared: held for the view's life - if (canWrite(principal, session)) return { - variant: "editor", - notes: store.notes, // the store's readable, exposed as-is - add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx), - remove: (args, ctx) => store.remove(args, ctx), - }; - return { variant: "reader", notes: store.public }; - }); -}); -``` - -:::flow wide | One store, three views. The redaction runs once in the store; both readers read the same value. -row "Per-principal views (remote)" blue -box ve "Notes { S1 }" "owner · editor" blue -box vr1 "Notes { S1 }" "guest Ann · reader" blue -box vr2 "Notes { S1 }" "guest Bob · reader" blue -row "Shared store (local)" accent -gap -box store "NoteStore { S1 }" "notes, public (a derive)" accent -gap -arrow ve -> store "self.get" -arrow vr1 -> store "self.get" -arrow vr2 -> store "self.get" -note "Authority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host." -::: - -`self.get(NoteStore, { session })` in a factory declares a **reference** that the view holds for its whole life; Chord -binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables -directly. Owner, Ann and Bob now give three `Notes` objects, one `NoteStore`, one note list and one redacted list. - -Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. -Base code uses the same pattern with the same primitives; nothing about it is special to core. - -## Derived values, briefly - -`self.derive(name, sources, fn)` computes a value from readables and recomputes it when a source changes; viewers get -only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun -that matters: `stripAuthor` must return the *same* object for an unchanged note, or every commit costs the size of the -whole list. The tool for that is `mapShared`. - -## Arguments are checked at the provider - -Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write -JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to -the **provider's own** schema (unknown fields removed) and validated with it, whoever the caller is, and never with a -schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves -(`instanceof Uint8Array` plus an optional length bound), never walked. Stripping copies only when a field was removed; -the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come -from the provider. - -On the same host nothing is copied: arguments and results are passed by reference. Values are **immutable by -convention** (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes -values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9). - -:::review | Review: are local arguments validated? (Q15, open) -Design §5's table says `local` arguments are "neither validated nor copied"; §3 and R7 say arguments are validated -"for every caller". Review round 3 raised this as Q15 (apply R7 to `local` too, or keep `local` as the one exception), -and `layer-1.md` lists it as open. Until it is decided, write local methods so that they do not depend on either -answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view -checking it. `chord-next-walkthrough.md` concept 6 still describes the older rule ("validated only when the caller is -not a host principal"), which R7 replaced. -::: - -## Choosing between the two - -Use a `remote` type for what callers address: views, things with per-caller authority. Use a `local` type for what -exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides -(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a -host-lifetime object, chapter V). - -# Writing It Down {#documents lede="Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it."} - -Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker -restart. An object's data lives in **documents**: named JSON values the object owns, changed only in -**transactions**. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states -them for Chord. - -## Defining a document - -```ts -// notes/worker.ts -const NoteList = defineState("notes.list", { - schema: obj({ items: arr(Note) }), - version: 1, - initial: () => ({ items: [] }), - persist: true, // stored; default: memory only - checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64, -}); - -const Draft = defineState("notes.draft", { // one unfinished note per author - schema: obj({ text: str() }), - version: 1, - initial: () => ({ text: "" }), - keyed: true, // many per object, by key - persist: true, - checkpointWhen: () => true, // small: a full base per write -}); -``` - -`checkpointWhen` is required on every definition, with no default (R14, as in Durable since pi `5c1736f82`): it says -when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents -checkpoint on every write; growing ones after a number of deltas. - -A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire -type, not a document type. - -## Singleton and keyed - -```ts -host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); // THE note list of this object - const drafts = self.state(Draft); // all drafts of this object - … -}); -``` - -| | singleton (no `keyed`) | keyed | -| --- | --- | --- | -| how many | one per object | many per object, by a short key | -| loaded | at activation, kept in memory | on demand | -| read | `list.value` (synchronous) | `await drafts.get(key)`: the value or `undefined` | -| list | (it is one value) | `for await (const key of drafts.keys())` pages keys | -| expose as state | yes: the handle is a readable | no: expose a method or a singleton index | - -There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large -collection is a method returning pages. - -## Transactions - -All writes go through `self.tx(fn, ctx)`. Inside, `await tx.draft(handle, key?, { tags? })` returns a draft of the -current document, or a draft over `initial()` if it is absent. Drafts record operations (assignments, `push`, -`Object.assign`); asking for the same document twice in one transaction returns the same draft. `tx.retire(handle, -key?)` removes a document. There is no `tx.get`: reading is drafting, and a draft is an overlay over the current -value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document. - -```ts -saveDraft: ({ author, text }, ctx) => - self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx), -``` - -Shorthands for one document: `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (loads the document, then runs the synchronous `fn` on -its draft) and `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`). - -### Four patterns - -Two decisions (R12 and R13, both as in Durable) shape how transactions are written. **Drafts are revoked when the -transaction settles**, so a transaction returns at most plain values, never something taken from a draft. And -**there is no existence check inside a transaction.** - -```ts -// 1. Return data: write in the transaction, read after the commit. -await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx); -return list.value.items.length; // the committed revision, or a newer one - -// 2. Remove if present: retire of an absent document is a no-op. -await drafts.retire(author, ctx); - -// 3. Create with tags if missing: the draft creates it, only if it changes. -await self.tx(async (tx) => { - (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; -}, ctx); - -// 4. Branch on existence: read first, outside the transaction. -if ((await drafts.get(author)) === undefined) { … } -``` - -### Durable's rule - -Inside the callback, only `await tx.*`. No other objects, no network, no models, no processes, no second `self.tx`, no -reads past `tx`. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch -first; decide and write inside. - -```ts -// Wrong: a fetch inside the transaction holds the object's line. -await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx); -// Right: fetch, then write. -const title = await fetchTitle(url, ctx.signal); -await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx); -``` - -## What a commit does - -:::flow wide animate | The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit. -row "Inside self.tx" accent -box line "wait for the line" "per object" -box run "run the callback" "drafts record ops" accent -box prep "prepare" "Delta checks placed values" -box write "storage batch" "deltas or a base" accent -row "After the write" blue -box adopt "adopt" "pointer swap" -box local "notify locally" "once, with the ops" blue -box wire "send" "one batch per link" -box revoke "revoke drafts" "touching one throws" -arrow line -> run -arrow run -> prep -arrow prep -> write -arrow write -> adopt "stored" -arrow adopt -> local -arrow local -> wire -arrow wire -> revoke -note "An error or an abort before the storage write discards everything. Storage commits are serialized per host." -::: - -Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same -host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the -document again; chapter XVII shows how one commit touching several holds still arrives as one message. - -## Tags and deleting by tag - -Tags are typed tokens set when a document is created: - -```ts -export const SessionTag = defineTag("studio.session"); // base code, shared by every plugin -await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx); -// when the session is deleted -await store.retireByTag(SessionTag, "S1"); // every document tagged S1 -``` - -The design names the operation `store.retireByTag(tag, value)` without saying who holds the store; session delete is -one of the app decisions still open (#18, chapter XIX). - -## Who may persist - -`persist: true` is allowed only in `local` objects and `lifetime: "host"` objects: identities without a principal. A -`remote` object's identity contains the principal key, which changes with the access path and the grant revision. Its -stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type -level (`self.state` in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a -stable identity: put the documents into a local shared store"). This is the second reason for the shared store. - -What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as -their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else -(natives, derives, routines, references) is rebuilt on activation. - -## Facet documents - -A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as -preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the -object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served -object's documents. - -## Ownership across plugins - -No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and -methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is -created (as with Durable's `createConversation({ init })`), never written into core's documents. - -## Versions and migration - -A change of shape or meaning raises `version` and adds a pure `migrate(old, from)`: - -```ts -const NoteList = defineState("notes.list", { - schema: obj({ items: arr(NoteV2) }), // NoteV2 adds `pinned` - version: 2, - migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }), - initial: () => ({ items: [] }), - persist: true, - checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, -}); -``` - -Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed -ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only -transaction writes it. - -Chord does not walk values against schemas. Instead, each definition computes a **schema fingerprint** once -(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord -compares them: the same version with a different fingerprint fails the object `broken` ("schema changed without a -version bump"). So every structural change needs a bump, an additive one with a trivial `migrate`. The one exception -to "no value checks" (R8): the result of `migrate` and of `initial()` is validated against the new schema once, when -produced. A failed check, a throwing `migrate`, or old code meeting a newer document refuses the access, keeps the -stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on -you. - -## Storage - -Behind the store is a `DocumentStorage`, passed as `createHost({ storage })`; memory by default. The server has one -shared storage, each worker one for its session. - -```ts -interface DocumentStorage { - commit(writes: readonly DocumentWrite[]): Promise; // create/change/retire, atomic - find(address: DocumentAddress): Promise; - load(id: string): Promise; // version, value, deltasSinceBase - scan(query: { owner: string; definition: string }, limit: number, cursor?: string): - Promise<{ keys: string[]; next?: string }>; - byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>; - mintId(): Promise; -} -``` - -Chord ships a memory storage and a SQLite storage (over `@earendil-works/chord/sqlite`, the driver layer that moved -from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address -occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. -Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over -Durable's. Your own storage can prove itself with `registerDocumentStorageConformance(runner, name, withStorage)` from -`@earendil-works/chord/testing`. - -Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next -access. A storage error ends the store and closes the host; `onError` receives `{ kind: "storage_failed" }` and the -app decides (chapter I). - -## Kinds of state - -| Kind | For | API | -| --- | --- | --- | -| document | data the object owns | `self.state`, `self.tx` | -| derive | computed values | `self.derive` (chapter VIII) | -| source readable | external data: Durable views, files, logs | value plus ordered changes; feeds derives | -| paged method | large or queryable collections | a method returning pages (chat history) | -| plugin database | full-text search, large blobs | a native opened by its owner (chapter IX) | - -App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives -in Durable. - -:::review | Review: does an absent singleton read `undefined` or `initial()`? (Q11, open) -Design §9 says "whether a document exists is read outside the transaction (`handle.value` / `handle.get` give -`undefined` for an absent one)" and also "a singleton that does not exist yet reads `initial()` through its handle; -retiring resets it to `initial()`". Review round 3 proposes: singletons always read `initial()`, only absent keyed -documents read `undefined`, and singleton existence needs no API. This guide follows the proposal (`list.value` is -never `undefined`); it is not decided. -::: - -:::note | Built so far -Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel -has a working version with a JSONL storage, which the design rejected. -::: - -# Holding On {#references lede="References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code."} - -A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and -started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the -whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a -**reference** does; plugin code contains no retry loops and no request ids. - -## Three ways to hold - -| Where | API | Released | -| --- | --- | --- | -| a factory | `self.get(Type, input)` | when the object ends | -| a call | `await ctx.get(Type, input, { state? })` | when the call ends | -| outside objects | `await host.get(Type, input, { signal, state?, parkTimeout? })` | `ref.release()` | - -Use `self.get` for what an object needs its whole life (the view's store), `ctx.get` for what one call needs, and -`host.get` only in UI code, tests and entry points. `{ state: false }` binds for methods only, without receiving state. -A reference keeps its object alive while held. - -## The object lifecycle - -```text -(get) → activating → live → ending → ended - └→ failed (backoff) → activating … -``` - -**Activating:** Chord binds the declared references and waits until they are live and hydrated, loads singleton -documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object -goes live when all of that succeeded. Calls wait behind its **gate** until then. - -**Ending:** routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, -references are released. An object with `lifetime: "referenced"` (the default) ends when its last reference goes, -after the type's `linger` if it has one. A `lifetime: "host"` object (only for types without a required input) lives -as long as the host. A running, finishing or blocked job counts as a reference (chapter XI). - -## `ref.state` - -Every reference reports what is going on as a tagged value; listeners get each new one: - -```ts -type RefState = - | { kind: "loading" } // first hold, reconnecting, restarting - | { kind: "live" } - | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry - | { kind: "released" }; -``` - -| Situation | Calls | Values | -| --- | --- | --- | -| link drop | park until reconnected | readable, `loading` | -| object loading or restarting | park until live | readable, `loading` | -| rollout: provider ahead or installing | park until installed | readable | -| object `failed` | reject at once with the cause | `failed`, with cause and `retryAt` | - -Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 -s by default, `parkTimeout` per `get`), then fail `unavailable`. Otherwise a call ends only through its caller's -signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is -not resent: Chord cannot know whether it took effect. - -**Errors look like disconnects.** A consumer handles a failed call exactly as a dropped link: show the error, let the -user retry. Work that must not run twice carries its own idempotency key as data (Durable's `requestId`, a job's -`key`). - -:::steps | The worker restarts while a tab watches the notes -+++ Watching -The tab holds `Notes { S1 }`. `ref.state` is `live`; `notes.notes.value` has 40 notes. -+++ The worker goes away -The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its -hold is gone; the tab's reference goes `loading`. The 40 notes stay readable. -+++ A click meanwhile -Ann's colleague clicks **Add**. The call **parks** on the reference; nothing is sent yet. The deadline is 60 s. -+++ Chord resolves again -Holds on S1 still exist, so Chord calls `route(Session).resolve({ id: "S1" })` again, with the per-address backoff. -The app forks a new worker: a new **incarnation** of `Session({ id: "S1" })`. -+++ Re-hold -The server re-holds every hold on S1 over the new link. The new worker activates `NoteStore { S1 }` (its notes load -from storage) and the views; it sends fresh snapshots. The tab's reference goes `live`. -+++ The parked call runs -The parked `add` is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code. -::: - -## Failure and retry - -Any broken part fails the **whole** object: a half-working object could break rules that rely on its documents -changing together. Chord tags the cause by where it failed: - -```ts -type FailureCause = - | { kind: "transient"; error: ChordError } // an open threw, a dependency was unreachable: automatic backoff - | { kind: "broken"; error: ChordError }; // factory, derive, hydration or migrate threw: retried only by a trigger -``` - -**Transient** causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success -resets it. **Broken** causes would fail the same way again (a code bug, invalid data), so they are retried only by a -**trigger**: a plugin reload, `ref.retry()` (a Retry button), or a readable the factory declared with -`self.retryOn(...readables)`. Only an opener may declare its own failure broken, with `throw broken(error)`, for -example for invalid configuration. A failed reference stays held; on success it goes `loading`, then `live`. - -```ts -host.serve(EnvConnection, ({ env }, self) => { - const config = self.get(EnvConfig, {}); // a facet or registry value, outlives the attempt - self.retryOn(config.envs); // a config change retries a broken open - const ssh = self.open(async ({ signal }) => { - const c = config.envs.value[env]; - if (!c) throw broken(new Error(`unknown environment ${env}`)); - return connect(c, signal); - }, (s) => s.close()); - … -}); -``` - -`retryOn` sources must live independently of the attempt (facet documents, registry values); a member of one of the -object's own references is a type error, since it goes away with the attempt. - -## Waiting for dependencies - -An object whose references are not live yet stays `loading` and goes live the moment they are, bounded by -`readyTimeout` (60 s). The decision for the failure case (R18) is what makes recovery automatic: - -:::sequence animate | Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live. -actor tab "tab" -actor view "Notes { S1 }" blue -actor store "NoteStore { S1 }" accent -tab -> view: hold -view -> store: self.get (activation) -store -> store: open fails: disk not mounted -store --> view: failed (transient) -view --> tab: failed: transient, cause inside -note view..store: the view keeps its reference; its state is the retry trigger -store -> store: backoff retry: open succeeds -store --> view: live -view -> view: retried at once (backoff cancelled) -view --> tab: live -::: - -A dependency that fails, or does not become live in time, fails the dependent **transient** with the dependency's -cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord -uses their state as a retry trigger. Factory authors write `self.get` and know none of this. After an object is live, -a dependency leaving `live` does not change it: its calls that use the dependency get the dependency's error, or park -while it loads. - -**Cycles are rejected** (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. -Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and -across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation -`broken`, naming it ("A → B → A"), and keeps no references. - -## No optional references - -`get` has no `optional` flag. When a dependency's plugin may be missing, turn the dependency around: the dependent -owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is -optional, so radius owns `radius.forwards` and ports contributes its forwards; without ports the table is empty and -radius works (chapter XIII). - -## `ctx` - -Every call gets a `ctx` as its last argument: - -| Field | Meaning | -| --- | --- | -| `signal` | fires when the caller cancelled or the object ended; pass it to everything you wait on | -| `principal` | the caller (remote types only) | -| `incarnation` | the calling host's incarnation, for calls that arrived over a link (chapter X) | -| `get` | references released when the call ends | -| `telemetry`, `trace(name, attributes, fn)` | the current span; a child span with a child `ctx` (chapter XVII) | -| `jobId`, `progress` | in a job's `run` (chapter XI) | - -```ts -exportText: async ({}, ctx) => { - const catalog = await ctx.get(Catalog, {}); // released when the call ends - const title = catalog.sessions.value[session]?.title ?? session; - const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal }); - return await res.text(); -}, -``` - -There are no code versions and no request ids in `ctx`. Durable's `Context` is the same module (`chord/context`): -`ctx` extends it, so passing `ctx` into pi-durable needs no conversion. Code outside objects gets a `ctx` from -`host.root(fn, { principal?, signal? })`; inside Durable tools it uses `chord(fn, context)` (chapter XIV). - -:::note | Built so far -References, `ctx` and the lifecycle are layers L4 and L5. `ChordError` and `FailureCause` are specified in the L1 -draft (`layer-1.md`), with codes `invalid`, `contract_mismatch`, `forbidden`, `gone`, `not_ready`, `released`, -`too_large`, `unavailable` and `aborted`, and a finer `reason` such as `invalid/args`. -::: - -# Taking Access Away {#revocation lede="Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back."} - -The owner removes Ann from S1. Her tab still has a WebSocket open and a `Notes { S1 }` view on worker S1. Both must end -now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the -mechanism; the app decides who is blocked and remembers it. - -## Labels, not predicates - -The app declares, once, which **revocation labels** each principal carries. Revoking means revoking one label; -everything carrying it is hit. - -```ts -// studio/src/principal.ts -export type Revocation = { grant: string } | { device: string } | { session: string }; - -export const revokedBy = (p: P): Revocation[] => - p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] - : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }] - : []; - -const server = createHost({ address: Server(), principal: { kind: "server" }, revokedBy }); -// every worker is created with the same revokedBy -``` - -| Principal | Labels | -| --- | --- | -| guest Ann (grant g1) in S1 | `{ grant: "g1" }`, `{ session: "S1" }` | -| guest Bob (grant g2) in S1 | `{ grant: "g2" }`, `{ session: "S1" }` | -| owner on the paired phone | `{ device: "phone" }` | -| owner, local | none | - -| Call | Hits | -| --- | --- | -| `server.revoke({ grant: "g1" })`: remove Ann | Ann | -| `server.revoke({ session: "S1" })`: archive S1 | Ann and Bob | -| `server.revoke({ device: "phone" })`: sign out the phone | the phone | - -A label is plain data because revocation is an event that must travel from the server to the workers; a predicate -function could not be sent. - -## What `revoke` does - -`host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by -canonical JSON; never the host's own principal), and passes `r` to its children, which apply their own `revokedBy`. - -:::sequence animate | Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record. -actor owner "owner tab" -actor server "server" accent -actor worker "worker S1" blue -actor ann "Ann's tab" "guest g1" muted -owner -> server: Sharing.remove({ grant: "g1" }) -server -> server: app: mark g1 blocked (document) -server => server: revoke({ grant: "g1" }) -server -> ann: close link -server => worker: revoke { grant: "g1" } -worker -> worker: end Notes { S1 } as g1 -ann -> server: reconnect -server -> server: authenticate: g1 blocked -server --> ann: refused (4401) -::: - -On worker S1, for `revoke({ grant: "g1" })`: - -| Object | Principal | Hit | -| --- | --- | --- | -| `Notes { S1 }` | owner, local | no | -| `Notes { S1 }` | guest g1 | **yes**: ends | -| `Notes { S1 }` | guest g2 | no | -| `NoteStore { S1 }` | none (local) | no | - -Ending Ann's view releases its reference to the store, which keeps serving the others. - -## Chord persists nothing here - -`revoke` cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which -`authenticate` checks on every new connection. After a server restart nothing live exists to revoke; every client -authenticates again and blocked grants are refused there. - -**No re-stamping.** A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, -her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references -re-hold by themselves, now as an editor. Reconnecting never restores revoked authority. - -**Revocation beats jobs** (R11). Normally a running job keeps its object alive after the last reference is released -(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the -abort signal and the normal close timeout. - -## Work in shared stores - -Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs *on -behalf of* someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by `revoke`. -Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine -(chapter X): - -```ts -// in a shared store that runs work for guests -const sharing = self.get(Grants, {}); // the sharing plugin's live grants -self.routines.on(sharing.live, async (live, ctx) => { - for (const run of shell.running.value) - if (run.args.grant && !live[run.args.grant]) shell.abort(run.id); // idempotent -}); -``` - -There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state -cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs. - -## Who does what - -| Chord | The app | -| --- | --- | -| stamps principals on links | what a grant is; share links, cookies, `authenticate` | -| keeps one object per principal key | defines `Revocation` and `revokedBy` | -| on `revoke(r)`: ends matching objects and links, passes `r` down | decides when to revoke; stores who is blocked | - -# Computed Values {#derives lede="Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking."} - -The reader's notes without authors, a note count, the number of people viewing: values computed from other values. -Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A -**derive** is computed once per change and sent to viewers as a difference. - -## `self.derive` - -```ts -const count = self.derive("count", [list], ([l]) => l.items.length); -const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({ - total: l.items.length, - pinned: pinned.length, -})); -``` - -`self.derive(name, sources, (values, previous) => value)`. Sources can be documents, readables, other derives and -state members of references. The value is recomputed when a source changes and diffed into operations for the wire. -Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer -never sees a note list and a count from different commits. - -`previous` is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a -host teardown, so a derive must be correct without it. A derive that throws fails its whole owner `broken`. - -## How the difference is found - -Chord compares the new value with the previous one using Delta's `diffRevisions`. Parts that are the **same objects** -as last time cost one `===` each. Parts that are new objects must be compared by content. - -**The footgun.** A derive that builds new objects for unchanged data makes every commit cost the size of the whole -value, per view. The reader list from chapter IV, written naively: - -```ts -self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) })); -// one note added to 2,000: 2,001 new objects, every one compared by content -``` - -**`mapShared(f)`** keeps one result per source element: `f` runs only for new or changed elements, unchanged ones -return their previous result object. - -```ts -import { mapShared } from "@earendil-works/chord/delta"; - -const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at })); -self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); -// one note added: one new object, 2,000 identical references; the diff emits one insert -``` - -:::review | Review: mapShared is planned, not available -Design §12 introduces `mapShared` as coming "from `@earendil-works/chord/delta`", but nothing in `packages/chord/src` -defines it yet, and Delta does not yet treat bytes as atomic leaves in `diffRevisions`. Design §28 lists both as the -step "Delta in pi", before the layered rebuild. Until then, keep a `Map` from source element to result yourself, which -is what `mapShared` will do. -::: - -## Cookbook - -| Case | How | -| --- | --- | -| select a field, look up by key, a small aggregate | compute it; the value is small | -| filter or map a list | `mapShared` over the elements | -| redaction | preferably separate documents per visibility; otherwise `mapShared`, computed once in the shared store, never per view | -| join two sources | `mapShared` plus the looked-up source's revision | -| a window over a large list | take a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends | -| a large view combined from several sources | re-path the sources' operations, as Durable's `ConversationView` does; application code never interprets ops | - -**Container width** (R24): structural sharing is O(change) along a path, not across a container's width. Changing one -item of a flat array copies the array's slots, and `mapShared` visits every element once (one identity comparison -each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented -limit. - -## Readables that are not documents - -Three small readables round out the toolkit: - -- `constant(value)`: never changes. Start data uses it (chapter IX). -- `self.writable(initial)`: a host-local readable of any in-process value (functions included), compared by identity, - with `set`. It never goes on the wire; it exists for registry contributions that change (chapter XIII). -- Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and - Chord v2 share the readable type (`chord/src/state.ts`), so it can feed a derive or be exposed as state directly. - -:::review | Review: exposing a writable (Q14, open) -A `writable` may hold functions; nothing yet stops a factory from returning one as a state member, which would reach -the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked -when the object is served. Undecided. -::: - -## Presence: who is looking - -`host.references(Type)` is a readable of who holds references to objects of a type on this host: input, principal and -client. Client holds carry an opaque origin id, so two tabs are two clients. - -```ts -host.serve(NoteStore, ({ session }, self) => { - const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) => - refs.filter((r) => r.input.session === session).length); - return { …, viewers }; -}); -``` - -A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the -UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There -is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's `linger` keeps a -just-closed object warm, and that also keeps the worker alive through `host.activity`. - -**"Leaving" is never derived from an unmount.** A UI reload unmounts and remounts components; a component unmounted by -a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, -viewer and login-abandon leases with one mechanism. - -:::note | Built so far -Readables and derives are layer L2 (a readable is today's `replicatedState`, derives reuse today's derived cells over -it). Not started in the rebuild. -::: - -# Things the OS Gives You {#natives lede="Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads."} - -Studio's notes get an inbox: a text file dropped into the session's `inbox/` directory becomes a note. That needs a -file watcher, a **native**: an OS-level value (a file handle, a socket, a process, a database connection) that cannot -be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data -directory, which the plugin does not know. This chapter covers both. - -## Start data is an object - -A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start -message. There is no special mechanism for them. Start data is an ordinary `local` object with `lifetime: "host"` and -constant state, served by base code's core facet: - -```ts -// studio/src/core/contract.ts -export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) }); -export const SessionInfo = local("studio.session-info", { - version: 1, - input: obj({}), - members: { init: Init }, -}); - -// studio/src/core/worker.ts -export const coreWorkerFacet = (init: Static) => defineFacet((host) => { - host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" }); - // … the Harness and other core objects (chapter XIV) -}); -``` - -Any plugin on the worker declares a reference and reads it **after activation**: in an opener, a derive, a routine or -a method, never synchronously in the factory. - -```ts -const info = self.get(SessionInfo, {}); -// later: info.init.value.id, info.init.value.dataDir -``` - -Paths are derived by the plugin from generic start data (`join(dataDir, id, "notes-inbox")`), never provided per -plugin. Platform bindings work the same way: on a Durable Object, a core object's `self.open` returns the binding -(`state`, `env`) and its close does nothing. - -## `self.open` - -```ts -const watcher = self.open( - async ({ signal }) => watchDir(path, { signal }), // open: during activation - (w, reason) => w.close(), // close: when the object ends or restarts - { closeTimeout: 10_000 }, // optional; host default 30 s -); -watcher.value; // the watcher; reading it before activation throws not_ready -``` - -`self.open` returns a handle with `.value` (type `Native`), not a proxy. The rules: - -- **Natives live only in objects.** Not at module level, not in the facet. A module-level watcher would survive a - reload with nobody to close it. -- **Order:** natives open one after another in declaration order (a later opener may read an earlier handle's - `.value`) and close in reverse. The object goes live after all opens succeeded. -- **The value never changes during an attempt.** No replacement, no versions, no stale-handle logic. To change a live - native, the object restarts. -- **A failed open** makes the object `failed`: transient with backoff by default, broken if the opener says so - (`throw broken(error)`). A partly failed activation closes the natives already open, with reason `end`; the next - attempt opens all of them anew. -- **`close` gets a reason:** `{ kind: "reload" }` (the plugin reloads and this native object restarts), `{ kind: - "stop" }` (the host shuts down) or `{ kind: "end" }` (the last reference ended, or the type was removed). Each - `close` is bounded by `closeTimeout`, then reported as a possible leak while the rest continues. A throwing `close` - does not stop the others; `host.close()` collects the errors, background ends report to `onError`. -- **Behaviour around a native goes into routines** (chapter X). Listeners on natives outside routines are forbidden. - -An object with at least one `self.open` is a **native object**; one without is a **pure object**. The difference -decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart. - -## The inbox - -The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a `lifetime: "host"` -object. The facet cannot `self.get` a type it serves, and the inbox has no meaningful input, so it is an object -without input that reads its session from the start data: - -```ts -// notes/contract.ts -export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} }); - -// notes/worker.ts -host.serve(NoteInbox, (_input, self) => { - const info = self.get(SessionInfo, {}); - const watcher = self.open(async ({ signal }) => { - const { id, dataDir } = info.init.value; // read in the opener: after activation - return watchDir(join(dataDir, id, "notes-inbox"), { signal }); - }, (w) => w.close()); - - self.routines.loop("import", async (ctx) => { // chapter X - const store = await ctx.get(NoteStore, { session: info.init.value.id }); - for await (const file of watcher.value.files(ctx.signal)) { - await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx); - await rm(file); - } - }); - return {}; -}, { lifetime: "host" }); -``` - -Note `ctx.get(NoteStore, …)` in the routine rather than `self.get` in the factory. The factory runs before the -session id is known (start data is read after activation), so a host-lifetime object without input gets -session-scoped objects with `ctx.get` in a routine or method (R22). A `get` whose input is computed after activation is -to be added only if a real case needs it. - -## Natives with work in flight - -A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, -use one of two patterns: - -- **Drain in the background.** `close` detaches the old resource and returns at once; it disposes of it when its work - finishes, or after `closeTimeout` at most. Chord opens the new one immediately. Code mode's sandbox does this: a - script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn - fails), new scripts start on the new one. -- **Swap through a registry.** The resource is a registry contribution (chapter XIII). Readers always look up the - current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline. - -Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a -sandbox) is the owner's business: `close` may drain instead of tearing down. What an old native does after its owner -restarts is also the owner's `close`: a model runtime keeps working until collected, a closed SSH connection fails its -in-flight `exec`. - -## Natives that follow a relocation - -A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be -heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its -internal resource and bumps a generation, so late events from the old location are dropped: - -```ts -const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close()); -self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox"))); -``` - -## Call-scoped natives - -A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a -download) is opened and closed inside the method with `try/finally`. There is no `ctx.open`: - -```ts -list: async ({ path }, ctx) => { - const session = await ssh.open(config, ctx.signal); - try { return await session.readDir(path, ctx.signal); } - finally { session.close(); } -}, -``` - -## Core vs. plugin - -`SessionInfo` comes from base code, `NoteInbox` from a plugin; both use the same primitives. Where code lives decides -which it is. **Base code** (the app's `src/`, imported statically by the start code under a fixed name such as -`@studio/core`) is always there, not switchable, and a change restarts the hosts. **A plugin** (its own directory, -found by listing, named by its `package.json`) is switchable and reloads. Core is what the app always needs and -everything builds on: the harness, the session list, access, the session location. Optional features are plugins; -code mode is a plugin. - -:::note | Built so far -Natives and host-lifetime objects are layer L4. `constant` is decided (R23), not built. -::: - -# Standing Behaviour {#routines lede="Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction."} - -Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived -session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare -`setInterval`s and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord -has one declared form for it: the **routine**. - -## Four kinds - -```ts -self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx)); // due time survives reloads -self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx)); // a readable or event source -self.routines.loop("import", async (ctx) => { // runs until the object ends - while (!ctx.signal.aborted) await importNext(ctx); -}); -self.routines.custom("ticker", ({ signal, run }) => { // anything else - const timer = setInterval(() => run(tick), 1000); - return () => clearInterval(timer); // cleanup undoes the setup -}); -``` - -Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their -plugin. Each run gets its own `ctx`; errors are caught and reported, never uncaught. Routines are named (`on` -optionally by its source) so Chord recognizes them across reloads; that is how `every` keeps its due time. - -## Serialized per routine - -The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at -most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported. - -The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through -`restart(W1)` when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and -restarts it a second time. - -## Reconcile, don't track - -A routine **reconciles from the current value**: it makes the world match what it sees now. It writes no "in -progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value -compares with the current value itself; there is no `overlap` option. Subscribing to the object's own documents is -gapless: every commit is seen. - -No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an -object's behaviour when it ends or reloads. - -## Rules between two owners - -Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived -session are read-only" involves the catalog (which knows archiving) and `NoteStore` (which holds notes). Two tempting -designs are wrong: - -- **A cross-owner transaction** (archive and lock the notes atomically) would make the catalog know every plugin and - couple their storage. -- **A notification** (`catalog` calls `notes.archive(session)`) can be lost: the worker is not running, the link drops, - the process crashes between the archive and the call. - -The design's rule: **the owner of the dependent data enforces the rule** by reconciling from the other owner's state -with an idempotent routine. - -```ts -// NoteStore: an archived session's notes are read-only -const flags = self.state(NoteFlags); // { archived: boolean }, NoteStore's own document -const catalog = self.get(Catalog, {}); // the server's session list: another owner -self.routines.on(catalog.sessions, async (sessions, ctx) => { - const archived = sessions[session]?.archived ?? false; - if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx); -}); -// add and remove refuse while flags.value.archived -``` - -After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The -same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked -(chapter VII), and keeps any invariant between plugins. - -## Reading without starting a worker - -The session list in the sidebar shows each session's note count. Getting `NoteStore { S1 }` for every session would -start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker -side reports a summary to a server-side object by a method call from a routine, and clients read the server object. - -```ts -// notes/contract.ts -export const NoteCounts = remote("notes.counts", { - version: 1, input: obj({}), placement: () => Server(), - variants: { - owner: { state: { counts: map(int()) } }, - worker: { methods: { report: method(obj({ session: str(), count: int() })) } }, - }, -}); - -// notes/server.ts -const Counts = defineState("notes.counts", { - schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }), - version: 1, initial: () => ({ bySession: {} }), persist: true, // survives a server restart - checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, -}); - -host.serve(NoteCountStore, (_input, self) => { // local, lifetime "host" - const counts = self.state(Counts); - return { - counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)), - report: ({ session, count, incarnation }, ctx) => - counts.change((d) => { - const last = d.bySession[session]; - if (last && last.incarnation > incarnation) return; // uuidv7: an older worker, ignore - d.bySession[session] = { count, incarnation }; - }, ctx), - }; -}, { lifetime: "host" }); - -host.serve(NoteCounts, (_input, principal, self) => { - const store = self.get(NoteCountStore, {}); - if (principal.kind === "worker") return { - variant: "worker", - report: ({ session, count }, ctx) => { - if (session !== principal.session) throw forbidden(); // a worker reports only itself - return store.report({ session, count, incarnation: ctx.incarnation }, ctx); - }, - }; - return { variant: "owner", counts: store.counts }; -}); - -// notes/worker.ts, inside NoteStore -const counts = self.get(NoteCounts, {}); // a reference to the server -self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx)); -``` - -The server object keeps the latest value per session (in a document if it must survive a server restart) and -**ignores reports from an older worker incarnation** (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the -fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): `ctx.incarnation`, -next to `ctx.principal`, not part of the principal or its key, never sent as data, so it cannot be forged. Session -status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no -caller incarnation of its own, so the view passes it on as data. - -:::review | Review: `ctx.incarnation` across a relay (Q13, open) -A worker's call to the server arrives directly, so `ctx.incarnation` is the worker's. But a call relayed through the -server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's -handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp -the principal. Undecided; the pattern above only needs the direct case. -::: - -:::note | Built so far -Routines are layer L8. The design treats `every` as one of four forms over one runner; review round 3 proposed cutting -to `on` and `loop` only (C4), not decided. -::: - -# Work That Outlives the Call {#jobs lede="Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome."} - -Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be -cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do -this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work -that must resume after a crash. Chord's form for it is the **job**, built in pi-durable's sandwich: **intent** (a -transaction), **effect** (anything), **outcome** (a transaction in the current code). - -## Defining and running a job - -```ts -// notes/worker.ts -const ImportJob = defineJob("notes.import", { - version: 1, - args: obj({ path: str() }), - progress: obj({ lines: int() }), - result: obj({ added: int() }), - concurrency: 1, // one import at a time per NoteStore -}); - -const ImportResults = defineState("notes.import-result", { - schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }), - version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true, - checkpointWhen: () => true, -}); - -host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); - const results = self.state(ImportResults); - const imports = self.job(ImportJob, { - run: async ({ path }, ctx) => { // 2. effect - let lines = 0; - for await (const line of readLines(path, ctx.signal)) { - await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx); // keyed by job id - ctx.progress({ lines: ++lines }); - } - return { added: lines }; - }, - finish: async (outcome, tx) => { // 3. outcome, current code - (await tx.draft(results, outcome.id)).outcome = - outcome.kind === "done" ? outcome.result : { error: outcome.error }; - }, - }); - return { - …, - imports: imports.running, // live list of runs - startImport: ({ path, requestId }, ctx) => - imports.start({ path }, ctx, { key: requestId }), // 1. intent (refined below) - abortImport: ({ id }) => imports.abort(id), - }; -}); -``` - -`defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })`; `progress` and the outcome -are typed by the schemas. - -:::sequence animate | The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code. -actor tab "tab" -actor store "NoteStore { S1 }" accent -actor run "run" "effect" blue -tab -> store: startImport({ path, requestId }) -store => store: 1. intent: busy check, dedupe by key, commit -store --> tab: { id } -store -> run: start after the commit -run -> run: read lines, add notes (keyed by jobId) -run --> store: progress { lines } (throttled) -note tab..store: the tab may close here: the job counts as a reference -run --> store: settles: { kind: "done", result } -store => store: 3. record outcome, finish(outcome, tx), ack -::: - -## The three steps - -**1. Intent.** `job.start(args, ctx, { key? })` is shorthand for `tx.start(job, args, { key? })` inside a -transaction, which may contain other writes. The busy check (`concurrency`) and deduplication (the same key returns the -existing run) are atomic and in the same commit; `run` starts only after the commit. Run ids are uuidv7. - -**2. Effect.** `run(args, ctx)` may do anything: processes, network, other objects. It writes no document itself; it -returns the outcome. External writes carry a dedupe key derived from `ctx.jobId` (above: note ids), so an interrupted -attempt cannot apply them twice. On a reload `run` keeps its old code. - -**3. Outcome.** When `run` settles, Chord records the outcome in its own record, then calls the **current** code's -`finish(outcome, tx)` and acknowledges the run in the same commit. The outcome is tagged: - -```ts -type Outcome = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" }); -``` - -A throwing `finish` changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. `finish` -is optional and follows Durable's transaction rule. - -## Dedupe ends with the run - -Deduplication covers **unsettled runs only** (Q9). Once `finish` has committed, Chord drops the run's record and key -and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the -owner's. Studio records which run a request started, in the same commit as the intent, and `finish` records the -result by run id (`ImportResults` above): - -```ts -const Requests = defineState("notes.import-request", { - schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true, - checkpointWhen: () => true, -}); - -startImport: async ({ path, requestId }, ctx) => { - const prior = await requests.get(requestId); // read outside the transaction (R13) - if (prior) return { id: prior.run }; // started before, maybe finished long ago - return self.tx(async (tx) => { - const run = await tx.start(imports, { path }, { key: requestId }); // dedupes a concurrent retry - (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id; - return { id: run.id }; - }, ctx); -}, -``` - -Two concurrent retries both miss `prior`, but the key makes `tx.start` return the same unsettled run. The client -watches `imports` for progress and reads the result by run id once it is gone from `running`. - -Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a -retention limit would let a retried key rerun its effect. The owner knows which results matter. - -## Watching, waiting, aborting - -- **`running`** is a readable of `{ id, version, args, progress, phase }`. `phase` is `running`, `finishing` (outcome - recorded, `finish` not yet committed) or `blocked` (with the reason). `concurrency` counts only active runs. The - panel shows a progress bar from it. -- **Progress** is in memory, throttled and sent as deltas. For output, use `OutputBuffer` and `Progress` from - `@earendil-works/chord/output`: bounded windows, adaptive throttling, no write amplification. -- **Waiting:** `start` returns a run with `outcome(ctx)`. The caller's abort ends only its wait, never the run. -- **Abort is a request:** `job.abort(id)` fires the run's `ctx.signal`; the run decides what it returns (a result - that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. `finish` runs in - Chord's own context. - -## Lifetime, persistence, revocation - -**Jobs keep their owner alive.** A running, finishing or blocked job counts as a reference to its object, which ends -only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also -count as activity for the worker's keep-alive. - -**Revocation takes precedence** (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs -get the abort signal and the normal close timeout. A job runs as its object's principal. - -**Persistence:** jobs are in memory by default and gone with the process. `persist: true` (only for stable -identities) keeps the record: a run live at a crash ends `lost` at the next activation and `finish` records it. -Nothing is rerun automatically. - -**Versions:** a run started under an older job version keeps its version; `finish` and `running` see it through pure -`accepts` adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the -reload: the old outcome is `blocked`, kept and reported until a later reload brings the adapter. - -**Native objects:** a job in a native object ends `aborted` when the object restarts on a reload (chapter XV). In a -pure object it keeps running on its old `run` code. - -## What is not a job - -| Work | Use | -| --- | --- | -| request and response that may die with its caller | a method | -| input into running work (a pasted code, an approval) | a flow (chapter XII) | -| work that must resume after a crash | a Durable task | -| standing behaviour | a routine | - -:::review | Review: jobs are the largest open simplification (#9, C3) -Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to -address and watch (`LoginFlow { id }`). Options: (a) jobs stay records inside their owner, watched through `running`; -(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: -shrink jobs to a runner over documents (no job versions, `accepts`, `blocked`, `persist`) or move them out of the -kernel. Not decided. `chord-next-walkthrough.md` concept 17 still shows the older outcome shape `{ ok: true, result }`; -the design tags outcomes by `kind`. -::: - -# Asking a Person {#flows lede="Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice."} - -A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's -approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any -tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a **flow** kept as its -owner's state, and the **Questions** service, a shared store for questions. - -## A flow is state, not a call - -The tempting design is a long-held server call, `step()`, that blocks until the next step. A link drop fails it, and -the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as **state**, which -re-holds by itself after any drop: - -```ts -// models/contract.ts (sketch from the design) -start: method(obj({ provider: str() }), obj({ id: str() })) // uuidv7 -answer: method(obj({ id: str(), step: int(), value: str() }), Answered) // ok | "step no longer active" -cancel: method(obj({ id: str() })) - -// LoginFlow { id }: state -type FlowState = - | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] } - | { kind: "done" | "failed" | "cancelled"; notices: Notice[] }; -``` - -The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A -promise-based callback inside the work (pi-ai's `prompt` during a login) stays **inside** the owner: its job keeps the -resolver, and nothing crosses an object boundary. - -Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a -second protocol: - -```ts -async function* steps(flow: LoginFlowRef, ctx: Ctx) { - let last = 0; - for (;;) { - const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx); - if (s.kind !== "running") return s.kind; - last = s.step!.id; - yield s.step!; - } -} - -for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) - await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); -``` - -`last` keeps steps from being lost or repeated across re-holds. - -:::review | Review: where a flow lives (#11, deferred) -Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object `LoginFlow { id }` -living until done; tied to #9 (jobs as objects). The `LoginFlow { id }` address above assumes (b). Also undecided is -how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped -reference, never an unmount (chapter VIII). -::: - -## The Questions service - -The Questions service (`apps/p2/docs/questions.md`) is a p2 design, not a Chord feature, built from Chord's -primitives: a shared store with per-principal views. The asker gets `ask`; viewers get the open questions and -`answer`. - -```ts -export type Question = - | { kind: "text"; message: string; secret?: boolean; help?: string } - | { kind: "select"; message: string; options: readonly { id: string; label: string }[] } - | { kind: "confirm"; message: string; allow: string; deny: string }; - -type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" }; - -ask(question, { - audience, // a readable or a list of principals - policy, - key?, // stable per asking flow: a re-ask reattaches to the pending question - deadline?, // then resolve with what is there; `missing` names who did not answer -}, ctx): Promise; -``` - -Its rules: - -- **The store is the truth.** Pending questions and their resolvers live in the store; `open` is a readable over it, - filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question. -- **One settlement point.** Register the waiter and the abort listener before publishing the question; an answer, an - abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer - active". Answers are validated against the stored question (a select id must be one of the options). Never await an - answer inside a transaction. -- **Policies over a live audience.** `first` resolves with the first answer; `quorum(n)` after n answers; `all` when - every member of the audience has answered. With `join: "live"` (the default), someone who joins becomes required and - someone who leaves drops out. -- **Authorization.** The asker is stamped by the host (a worker asks only in its own session's scope; secret questions - only from server askers). Each question stores who may answer it; a guest's send access does not imply approval - rights. Secret answers never appear in `open`, documents or logs. -- **Link drops and crashes.** A rejected `ask` is not a denial. Askers retry with the same `key` until their own signal - aborts; a pending `ask` counts as an active call for keep-alive. -- **Where it lives.** On the host where the asking work runs: session questions on the session's worker, login - questions on the server. UIs reach both through the usual relay. - -The store in Chord v2 terms: - -```ts -host.serve(QuestionStore, ({ scope }, self) => { - const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed")); - const open = self.derive("open", [pending.value.readable], ([all]) => all); // data only, no resolvers - return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) }; -}); - -host.serve(Questions, ({ scope }, principal, self) => { - const store = self.get(QuestionStore, { scope }); - if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) }; - if (!mayView(principal, scope)) throw forbidden(); - return { - variant: "viewer", - open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)), - answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx), - }; -}); -``` - -The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and -closed with it). Its `close` rejects every waiter; askers retry with their key against the new store after a reload. - -:::review | Review: questions.md predates the v2 rules -`apps/p2/docs/questions.md` writes the store as `self.resource(...)` with a `replace` hand-over, which the design -rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with `host.root`, which Q6 -replaced with `chord(fn, context)` (chapter XIV). Design §20 says the Questions store "must live in an object (it -predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering -per viewer in the store, once per variant, may be cheaper for large audiences. The `ask` signature passes typed -objects as arguments, which is fine (data), but the `audience` readable form cannot cross a link: across links, -audience must be data (principals) or computed by the store. -::: - -## Choosing - -| Case | Use | -| --- | --- | -| a multi-step login in one owner | a flow: the owner's job keeps the resolver, state shows the steps | -| a tool needs the owner's approval | Questions, `first`, audience the owner | -| the agent asks everyone present | Questions, `all`, audience from presence | -| deploy needs 2 of 3 approvals | Questions, `quorum(2)` | -| a notice (auth URL, device code, progress) | not a question: the asking work's own progress state | - -# Extension Points {#registries lede="The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge."} - -The notes plugin wants a download route, `GET /s/:session/notes.txt`. A logs plugin wants to time every request. -Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these -contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two -things pass: object types and **registries**. - -## The facet, completed - -The facet has appeared since chapter II as the place that serves types. It is also itself a **pure object** with the -host's lifetime: - -```ts -// notes/server.ts -export default defineFacet((host, self) => { - const prefs = self.state(NotesPrefs); // plugin-wide documents - self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } }); - host.serve(NoteCountStore, …, { lifetime: "host" }); - host.serve(NoteCounts, …); -}); -``` - -- It may hold documents, references, routines and contributions; **no natives** (those need an object with its own - lifetime). -- Its identity is the plugin's name, so its persisted documents survive restarts and reloads. -- Served objects go live after the facet and may read and write its documents (chapter V). -- `host.serve` and `self.contribute` are valid only while the setup runs. The facet never `self.get`s a type it - serves. - -## Registries - -A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor -contributes **at most one value** per registry; the table is an immutable, ordered list of `(contributor, value)`; -the registry's `compose` turns it into the value consumers use. - -```ts -// owned by the HTTP component -export const Http = defineRegistry("chord.http", { - version: 2, - accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, - compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value -}); - -self.contribute(Http, { routes: { … }, wraps: [ … ] }); // a contributor -const router = host.registry(Http).value; // a consumer: live -``` - -**`compose` runs at runtime, whenever the table changes:** after a reload's commit, when a contributor goes live or -ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native -is open), so no check before a reload could be complete. `compose` always delivers a value with its fallback built in -(e.g. the first by plugin name wins) and reports conflicts through `report`, which become owner notices and telemetry -events. Readers get a plain value and never handle conflicts. Only if `compose` throws is the registry in an error -state, and readers get an error when they use it. Work that started earlier keeps the value it read. - -**Rules:** - -- **Who contributes** (R21): only facets and `lifetime: "host"` objects; only their `self` has `contribute` (a type - error elsewhere). Every object may read registries. -- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an - object's from `live` until it ends, fails or restarts. -- **Order** is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. - Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: `keyed(table, entriesOf)`, - `ordered(table, itemsOf, orderOf?)`. -- **One authoritative declaration** (Q8): only the defining plugin's facet declares the registry to the host - (`compose`, version, `accepts`). Other plugins' tokens only identify it by name and types. A reload of the defining - plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the - registry has no value (readers get an error like a failed reference) and contributions wait in the table. -- **Versions** (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted - by the defining facet's pure `accepts`. One without a matching version or adapter is left out and reported naming - plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and - are not schema-checked. -- **Use sites guard contributions.** A contribution is untrusted code (third-party plugins are a goal). Every use site - catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others. -- **The owner decides** conflicts, order and selection. Registries add behaviour; they never replace an owner's checks. - -## Natives through registries - -Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; -consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models -plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload -the next request gets the new runtime, while a running stream finishes on the old one. - -A handler that stores `const r = entry()` in a module variable and uses it for days has a bug: after the contributor -restarts it holds a closed native. - -## Contributions that change - -Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed -for its contributor's life (dynamic parts behind handles read at use) or a **readable**, which Chord subscribes to; -on every change Chord replaces that entry and runs `compose` again. For in-process values, `self.writable(initial)` is -a host-local readable of any value (no JSON, no wire, compared by identity) with `set`: - -```ts -const ext = self.writable(mcpExtension([])); // a Durable Extension, no tools yet -self.contribute(HarnessExtensions, ext); -self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools))); -``` - -There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer -contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension -without tools, empty routes); the table keeps exactly one entry per live contributor. - -## Optional dependencies - -Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin -manages, but ports is optional. So radius owns `radius.forwards` and ports contributes its forwards; without ports the -table is empty and radius works. - -## The HTTP component - -HTTP is an optional Chord component (`@earendil-works/chord/next/http`) built on a registry. A route gets a plain -request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed -and capped JSON bodies, uploads in chunks) and a `ctx` whose `principal` is the request's, so `ctx.get(Notes, …)` gives -exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges -with 206 and 416, HEAD, download headers with a safe filename, `nosniff`, CSP and its nonce, caching, streaming, abort -on disconnect): - -```ts -type HttpResponse = - | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string } - | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise } - | { kind: "redirect"; to: string } | { kind: "error"; status: number }; - -// notes/server.ts -self.contribute(Http, { - routes: { - "GET /s/:session/notes.txt": async (req, ctx) => { - const notes = await ctx.get(Notes, { session: req.params.session }); // the requester's view - const text = new TextEncoder().encode(render(notes.notes.value)); - return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength, - read: async (offset, length) => text.subarray(offset, offset + length) }; - }, - }, -}); -``` - -Routes compose with `keyed`: two contributors with one pattern is a reported conflict, and the first by plugin name -wins. **Wraps** are middleware in phases the owner defines (`observe`, `headers`, `transform`, outermost first), -within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's -lifecycle phases and encapsulation are the model, not Express. - -```ts -// logs/server.ts -self.contribute(Http, { - wraps: [{ - name: "access-log", phase: "observe", scope: "all", // default scope "own": only own routes - wrap: async (req, next, ctx) => { - const t = Date.now(); - const res = await next(req); - log(req, res, Date.now() - t); - return res; - }, - }], -}); -``` - -`route` narrows a wrap to one route; a `route` that does not exist is a reported conflict. Wraps may add a security -boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. -Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. - -## Channels and big bytes - -**Channels** are WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links and -with their own acknowledged windows, so a large state message never holds up keystrokes. - -**Big transfers are chunked by the owner,** not by Chord: bytes are a schema type (`bytes()`), carried natively -(never base64), and a 50 MB upload is a protocol of plain calls: - -```ts -beginUpload({ name, size }, ctx) → { upload } -append({ upload, offset, bytes }, ctx) // about 1 MiB per chunk; the awaited call is the backpressure -finishUpload({ upload }, ctx) → { id } -read({ id, offset, length }, ctx) → bytes -``` - -After a link drop the `append` in flight fails like any call; the writer asks the owner for the stored offset and -resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules -for drops, cancellation, relays and restarts, which this offset protocol already survives. - -:::note | Built so far -Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the -kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the -playground serves its pages with plain app code. Their design stays the target. -::: - -# The Agent {#agent lede="pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people."} - -Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, -whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs -plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable -and shows only how it plugs into Chord. - -## Shared foundations - -Durable and Chord v2 share modules: Durable imports Chord's `context`, `delta`, `json`, `output` and `sqlite`, and v2 -imports the same, so `Context`, `Draft` and `JsonValue` are one type each. A v2 readable is today's -`replicatedState` (`chord/src/state.ts`), so Durable's `viewState()` and document watches can be exposed as Chord state -members without conversion. Only plain data crosses between Durable and Chord objects. - -## The harness is a core object - -```ts -// studio/src/core/harness.ts (base code) -import { Harness as DurableHarness } from "@earendil-works/pi-durable"; - -export const ChordHostKey = createContextKey>("studio.chord-host"); -export const HarnessModels = defineRegistry("studio.harness-models", { version: 1, compose: firstModels }); -export const HarnessEnvironments = defineRegistry("studio.harness-envs", { version: 1, compose: firstEnv }); -export const HarnessExtensions = defineRegistry("studio.harness-extensions", { - version: 1, - compose: (table, { report }) => extensionSnapshot(table, report), // one Durable Extension per plugin -}); -const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", { - version: 1, compose: onlyEntry, // core-private: one contributor -}); - -host.serve(Harness, (_input, self) => { - const info = self.get(SessionInfo, {}); - const harness = self.open(async ({ signal }) => { - const { id, dataDir } = info.init.value; - const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT)); - return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), { - models: () => host.registry(HarnessModels).value, // read at each use - env: (target) => host.registry(HarnessEnvironments).value(target), - settings: () => currentSettings(), - registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed value - }, base); - }, (h) => h.close(BACKGROUND_CONTEXT)); - self.contribute(HarnessRuntime, () => harness.value); // for core's conversation objects - return { … }; -}, { lifetime: "host" }); -``` - -- **Lifetime `"host"`:** it runs for the worker's whole life, also while nobody holds it, because Durable tasks - continue. -- **Base code:** it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host - restart) does. -- **Registries, looked up at each use:** Durable's `models` and `settings` accept functions (pi `7ec1f648e`), so each - generation gets the current value. After a models reload the next request uses the new runtime; a running stream - finishes on the old one. -- **Durable's storage is its own.** Chord documents never commit together with Durable transcripts. - -The design writes the call as `Durable.open(storage, { … })`; pi-durable's actual entry point is -`Harness.open(storage, options, context)`, used here as `DurableHarness` to keep it apart from the core object -`Harness`. - -## Start order - -At host start Chord loads every facet and waits until every facet and every `lifetime: "host"` object **other than the -harness** is live or failed: the **start barrier**. Then the registries hold the contributions of every plugin that -loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model -runtime, is therefore `lifetime: "host"`. Nothing waits for specific contributions; what is missing shows where it is -used: - -| Missing | What happens | -| --- | --- | -| a tool | its call gets an error result the model sees | -| a model | the generation reports "model unavailable" and waits, not fails, until a model appears | -| a task definition | the task stays blocked (`missing_task`) until the plugin is installed or the task is aborted | -| a plugin that failed | reported with its error; a facet in backoff contributes later and the registries recompose | - -The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and -hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must -show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then. - -:::review | Review: the harness exception to the barrier (C17, R17) -R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: -no harness-specific exception; the harness is an ordinary object the app's start code holds after `load` resolves. -Not decided. `chord-next-walkthrough.md` concept 28 still says the harness's activation *is* the start barrier, which -R17 replaced. -::: - -## Tools that call Chord - -Durable tools, hooks and sections receive Durable's `ToolExecutionApi` (or `HookApi`) and a `Context`, never a Chord -`ctx` (Q6). Core put the host into the harness's base context under `ChordHostKey`, and every phase context inherits -it. A core helper opens a Chord call scope from any Durable context: - -```ts -// studio/src/core/chord.ts -export function chord(context: Context, fn: (ctx: Ctx) => Promise): Promise { - const host = context.value(ChordHostKey); - if (!host) throw new Error("not inside the Studio harness"); - return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry }); -} -``` - -The scope runs as the host principal; attribution is data; references taken with `ctx.get` are released when `fn` -settles. There is no wrapping of tools and no app-specific `defineTool`. The notes plugin gives the agent a tool: - -```ts -// notes/worker.ts -const addNote = defineTool({ - name: "notes_add", // tools are named _ - description: "Add a note to this session's notes panel", - parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }), - execute: (args, api, context) => chord(async (ctx) => { - const info = await ctx.get(SessionInfo, {}); - const store = await ctx.get(NoteStore, { session: info.init.value.id }); - const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx); - return { output: [{ type: "text", text: `added note ${id}` }] }; - }, context), -}); - -export default defineFacet((host, self) => { - self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] })); - … -}); -``` - -A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so -a tool call already running finishes on the old code and the next phase uses the new one. - -## Tools that need a person - -Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's -state: the Questions service (chapter XII). The model-facing `ask_user` tool and an approval hook both call -`Questions.ask`; tabs render the open questions and answer them. - -```ts -hook(ToolTask, { - beforeTool: async (call, api, context) => { - if (!needsApproval(call)) return undefined; - const decided = await api.memo("approval", context); // a rerun does not ask again - if (decided !== undefined) return decided ? undefined : { block: "Denied" }; - const allowed = await chord(async (ctx) => { - const q = await ctx.get(Questions, { scope: { session } }); - for (;;) { - try { - const r = await q.ask( - { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" }, - { audience: approvers, policy: { mode: "first" }, key: api.taskId }, - ctx, - ); - return r.answer; - } catch (error) { - if (ctx.signal.aborted) throw error; // a link blip is not a denial: retry - await backoff(ctx); - } - } - }, context); - await api.memo("approval", allowed, context); - return allowed ? undefined : { block: "Denied by the user" }; - }, -}); -``` - -Durable runs `beforeTool` in the tool task's call phase; after the execute checkpoint a crash retries execution -without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the -approval hook after hooks that rewrite arguments. - -## Conversations for the UI - -Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 -design; one construction that follows the rules: a core `ConversationStore { conversation }` (local) opens the Durable -watch as its own native, getting the Durable harness from the core-private `HarnessRuntime` registry contribution (a -native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal -`Conversation { session, conversation }` views in front of it check who may send (chapter XIX). - -## Code mode - -Code mode is a plugin. It owns its `pi-codemode` sandbox as a native; its tool entry looks up the **current** sandbox -when `execute()` starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the -new sandbox. On `reload` and `stop` the old sandbox is sealed and drained in `close`: running scripts finish or hit -their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today. - -## What lives where - -| Data | Where | -| --- | --- | -| transcript-bound: entries, documents that fork and rewind with a conversation | Durable | -| app state: session list, notes, drafts, grants, settings | Chord documents | -| data core enforces on conversations (read-only one-shots, cwd followers) | a core per-conversation policy document, written when the conversation is created (`createConversation({ init })`) | - -**Telemetry:** pass the host's adapter as `HarnessOptions.telemetry`; Durable stores trace carriers on submissions and -tasks (chapter XVII). - -# Reload and Rollout {#reload lede="What happens when you press Reload: per host, per object, and across the server, the workers and every tab."} - -Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters -asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in -documents) exists so that this moment is safe. This chapter shows what Chord does with it. - -## On one host - -```text -1. load new bundles; an import or syntax error rejects -2. staging facet setups and every live object's new factory run with a recording self - (declarations only, nothing activated); any throw rejects -3. commit the host switches to the new code atomically -4. after per object: pure objects swap, native objects restart -``` - -**Rollback is all-or-nothing per host.** Any load or staging error leaves the whole host on its previous code, so a -host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed -open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory -trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with -`contract_mismatch` and a notice naming plugin and type. - -:::steps | Reloading @studio/notes on worker S1, object by object -+++ Load and stage -The new bundles load. The facet setup and the factories of `Notes` (three views), `NoteStore { S1 }` and `NoteInbox` -run with a recording `self`: they declare, nothing activates. Nothing throws, so the host commits to the new code. -+++ NoteStore: pure, swapped in place -`NoteStore` has no `self.open`. Its new methods and derives take over; `NoteList`, the drafts, its references and the -running import job stay. Routines restart, serialized: the new `cleanup` waits for the old handler in flight. The gate -opens. -+++ Running calls finish on old code -An `add` that was running when the swap happened finishes on the old code with its old handles; its transaction still -commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that -touches the migrated document throws, the document stays intact, and that call fails once. -+++ The import job keeps its old run -The job's `run` keeps the code it started with. When it settles, the **new** `finish` records the outcome. -+++ The views: pure, swapped -The three `Notes` views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with -the new sort and the difference is sent. Nobody sees a reconnect. -+++ NoteInbox: native, restarts -`NoteInbox` has a watcher. Its running work ends `aborted`, the watcher closes with reason `reload`, the new factory -runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the -watcher's business: Studio's `watchDir` scans the directory once when it opens. -::: - -**Why natives restart instead of being handed over:** a native has no line, no transactions and no schema. If old and -new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base -code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild -in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket -(relayed connections drop) hurts. - -**Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory documents -go with them; persisted ones stay until retired. **Consumers elsewhere** see a restarting object like any object that -briefly goes away: `loading`, parked calls, then `live`. - -**UI facets reload too.** A reload unmounts and remounts components. That is why "leaving" (abort a login, close an -overlay) is never derived from an unmount (chapter VIII). - -## Across hosts - -A **revision** is (server incarnation, sequence) and names the complete desired code set: every facet with its -digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it -runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), -so a server restart with unchanged code triggers no reinstall. - -The order is **server, workers, clients**. Each host reports `installed`, `installed_failed` (with the objects that -failed) or `rejected` (with the reason; it stays on its previous code). A server that rejects stops the rollout. A -host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized -per host and bounded (60 s). - -Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response -carries the provider's revision and status: - -| Provider | The call | -| --- | --- | -| ahead, or behind and installing | `updating`: values readable, calls park (with the park deadline) | -| rejected | fails at once; retried on the next revision | -| same revision | `contract_mismatch`: a real mismatch, final | - -A host checks its own install status first: if its own install was rejected, a call fails at once with the reason -instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. -There are no cross-host transactions and no votes. - -:::sequence animate | A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision. -actor tab "tab" "revision 8" -actor server "server" "revision 8" accent -actor worker "worker S1" "installing 8" blue -server => worker: install revision 8 -tab -> server: call Notes.add (new contract) -server -> worker: relay -worker --> server: updating (rev 7, installing) -server --> tab: updating -tab -> tab: park (deadline 60 s) -worker --> server: installed 8 -server --> tab: provider at 8 -tab -> server: call Notes.add (resent from park) -server -> worker: relay -worker --> tab: result -::: - -## Start barrier - -At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start -data among them), wait until every facet and `lifetime: "host"` object other than the harness is live or failed, then -resume dependent work. Plugin reloads need no barrier. - -## Loading and the dev loop - -- **Roots:** plugins come from built-in, global (`/plugins`), package (npm or git, installed with - `--ignore-scripts`) and project roots, found by directory listing; no source parsing. The server selects which load. - Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects `.`, - tools `_`. -- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet, with host - packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per - (facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static - imports (tests, Durable Objects) work the same, with the app passing the plugin name to `load`. -- **Dev loop:** a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web - shell, a CLI, an agent tool such as p2's `reload_plugins`) reloads the changed plugins, or restarts all hosts for - base code. Reloads are always explicit. - -:::review | Review: project plugins (#14, deferred) -Per-directory plugins with trust prompts and snapshots (`apps/p2/docs/plugin-loading.md`, still the reference for -roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not -settled. -::: - -:::note | Built so far -Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and -a development freeze, both removed from the design. -::: - -# Changing Contracts {#contracts lede="How a contract gets a new version while old tabs, old workers and old plugins are still talking to it."} - -Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout -finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's -web bundle. A plugin from another author was built against `Notes` version 1 and is not updated yet. In both, a -consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is -also the one part of Chord v2 that is built: everything in the sections on `evolve`, `checkAccepts` and fingerprints -runs today in `packages/chord/src/next/contracts/`. - -## What has a version - -Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way: - -| Surface | Version | When an old one shows up | Chapter | -| --- | --- | --- | --- | -| object type (contract) | `version` plus fingerprint | `evolve` and `checkAccepts` on the provider; otherwise `contract_mismatch` | this one | -| document | `version` plus schema fingerprint | pure `migrate` on access; a schema change without a bump fails the object `broken` | V | -| job | `version` | `accepts` on outcomes and `running`; `blocked` without an adapter | XI | -| registry | `version` | `accepts` adapts contributions; one without a match is left out and reported | XIII | -| native | none | the object restarts on reload | IX | -| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | -| link protocol | protocol version | refused; hosts update together, a web page reloads | XVII | -| UI components, Chord and app APIs | none yet | exposed as they are; an SDK will version them later | | - -The rule behind every row is the same as for documents: **every change bumps the version**, additive ones included. -Chord never guesses compatibility from the shape of two schemas. - -## A history, not a file of versions - -A contract history starts with a **base**: a full `remote()` or `local()` declaration with an explicit `version`. The -`Notes` contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a `pinned` -flag and editors get a `pin` method. Instead of editing the declaration, the plugin continues the history: - -```ts -// notes/contract.ts -import { bool, evolve, nul } from "@earendil-works/chord/next"; - -export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() }); -export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() }); - -export const NotesV2 = evolve(Notes, (e) => - e.state("editor", "notes", obj({ items: arr(NoteV2) })) - .state("reader", "notes", obj({ items: arr(PublicNoteV2) })) - .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()), -); -``` - -`evolve(prev, (e) => …)` appends one version to `prev`'s history: `NotesV2.version` is 2, and `NotesV2` still knows -version 1 exactly as it was declared. Every version is **materialized and fingerprinted at declaration**. That is what -makes old code safe: an old plugin bundle that contains its own copy of `Notes` version 1 computes the same -fingerprint as the history inside `NotesV2`. There is no build step, no generated file and no fingerprint copied by -hand. - -The edits: - -| Edit | Changes | Kind | -| --- | --- | --- | -| `e.state(variant, member, schema)` | one state member of one variant | remote | -| `e.addState(variant, member, schema)` / `e.removeState(variant, member)` | adds or removes a state member | remote | -| `e.state(member, schema)`, `e.addState(member, schema)`, `e.removeState(member)` | the same, flat | local | -| `e.args(name, schema)` / `e.result(name, schema)` | one side of a method | both | -| `e.method(name, args, result)` | both sides of a method | both | -| `e.addMethod(variant, name, args, result)` | adds a method (or an existing one to another variant, same signature) | remote | -| `e.addMethod(name, args, result)` | adds a method | local | -| `e.removeMethod(name)` | removes a method from every variant | both | - -Edits chain. The `method(...)` wrapper appears only in base declarations; edits take the schemas directly, so a -method without a result is added with `nul()`. The rules of chapter III still hold across versions: a method has one -signature in every variant, and a member's kind never changes, not even after it was removed and added again. - -**What `evolve` cannot do:** change the input or the set of variants. Both change what an object *is* (its identity, -who gets which members), and no adapter can turn one into the other. That needs a new history base, below. - -## The provider says what it accepts - -`NotesV2` says what changed. It does not say how version 1 consumers are served; the **provider** decides that, with -mappings that live in its own code, never in the contract. The worker facet declares them with `checkAccepts`: - -```ts -// notes/worker.ts -import { checkAccepts } from "@earendil-works/chord/next"; - -const toV1 = ({ pinned, ...note }: Static) => note; -const toPublicV1 = ({ pinned, ...note }: Static) => note; - -const notesAccepts = checkAccepts(NotesV2, { - 1: { - state: { - editor: { notes: (current) => ({ items: current.items.map(toV1) }) }, - reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) }, - }, - methods: {}, // add and remove did not change; pin is new - }, -}); -// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }] - -host.serve(NotesV2, notesFactory, { accepts: notesAccepts }); // working name for the objects-layer option -``` - -`checkAccepts` is **pure**: it checks the mappings without calling them and returns only the version metadata of the -old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later -compares a consumer's version and fingerprint against this list. - -The mappings point in the direction the data flows: - -| Member in the old version | Mapping | Direction | -| --- | --- | --- | -| state, changed | `(currentMember) => oldMember` | provider's current value to what the old consumer expects | -| state, removed | `(currentVariantState) => oldMember`, or `"refuse"` | receives the whole current state of that variant | -| method arguments, changed | `args: (old) => current` | old caller's arguments to current ones | -| method result, changed | `result: (current) => old` | current result back to the old shape | -| method, changed or removed | `{ call: async (oldArgs, ctx) => oldResult }`, or `"refuse"` | replaces the method for old callers | -| member added since | nothing | an old consumer never asks for it | - -A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one -can be computed from what the variant has now, or refused. `e.method(name, args, result)` replaces both sides and -therefore needs both mappings, even if one side stayed the same (then an identity mapping). - -**Required, at type level and again at runtime.** For every member that exists in the old version and changed or was -removed since, the mapping is required by the type of `checkAccepts`. Because plugins are bundled without type -checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version -or a wrong shape throws, naming the member's path (for example `accepts[1].state.reader.notes`). An absent `state` or -`methods` section counts as empty and may be left out only when nothing there needs a mapping. - -**Conservative with no-op edits.** An edit that sets a schema to what it already was, or changes a member and then -changes it back, still requires an identity mapping. TypeScript erases constraints such as `{ max: 2000 }` from the -static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell. - -### Old versions map straight to the current one - -Studio later replaces `remove` with `archive`, which keeps the note in a trash list: - -```ts -export const NotesV3 = evolve(NotesV2, (e) => - e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()), -); - -const notesAccepts = checkAccepts(NotesV3, { - 1: { - state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } }, - methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } }, - }, - 2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } }, -}); -``` - -Each entry maps **one old version directly to the current one**, not through the versions in between. Version 1 -needs the state mappings (its notes have no `pinned`) and the `remove` replacement; version 2 needs only the -`remove` replacement, and leaves `state` out because its state is unchanged. A member that an old version never had -needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is -deleting its entry: version 1 consumers then get `contract_mismatch`. - -## Binding an old consumer - -A hold carries only the consumer's contract **version and fingerprint**, never a description of the contract (R6). -The provider decides with its own schemas: - -:::flow wide animate | What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input. -row "The hold arrives" -box hold "hold" "version 1, fingerprint F1" blue -box cur "equal to current?" "version and fingerprint" -box acc "accepted old version?" "F1 from the provider's own history" -row "Outcome" accent -box live "bind" "current members" -box adapted "bind adapted" "through the mappings" accent -box mismatch "`contract_mismatch`" "or updating, during a rollout" muted -arrow hold -> cur -arrow cur -> live "yes" -arrow cur -> acc "no" -arrow acc -> adapted "yes" -arrow acc -> mismatch "no" -note "A mismatch fails only that consumer's binding. The provider object and its other bindings are not affected." -::: - -Once a version 1 tab is bound, it is served as if version 1 were current: - -:::steps | A version 1 tab on a version 3 worker -+++ Bind -The tab holds `Notes { S1 }` with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and -`notesAccepts` lists version 1, so the hold binds adapted. The tab gets the `editor` variant as version 1 defined it: -`notes`, `add`, `remove`. It never sees `pin` or `archive`. -+++ State through the mapping -Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value -and keeps the result per binding. The snapshot the tab receives has no `pinned` fields. -+++ A commit -The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, -Chord applies the mapping to the new value and sends the difference to its last adapted value (`diffRevisions`). The -mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent -whole on every commit. -+++ A call -The tab calls `remove({ id })`. Its arguments are checked against the provider's own version 1 schema, then the -`call` replacement runs `archiveNote` on version 3 code. Its `null` result is what version 1 expects. Had the method -only changed, `args` would have mapped the arguments forward and `result` the result back. -+++ The worker moves on -A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), -the hold gets `contract_mismatch`, and Studio's panel shows "reload to update" instead of failing silently. -::: - -## A new history base - -Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from -`{ session }` to `{ project }`. That is a new base, written in full, with an explicit version that continues the -published numbering: - -```ts -export const NotesV4 = remote("notes", { - version: 4, // explicit: the published history used 1 to 3 - input: obj({ project: str() }), - placement: () => Server, - variants: { - editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } }, - reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } }, - }, -}); -// checkAccepts(NotesV4).versions is empty -``` - -A new base has no history before its version and **cannot accept older versions**, not even for an additive change. -Every consumer built against versions 1 to 3 gets `contract_mismatch` and must update. Later `evolve` calls continue -from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or -copy its fingerprint; the contract objects are branded and `evolve` and `checkAccepts` reject copies. - -## Local contracts - -`local` contracts (chapter IV) evolve the same way, with a flat members map and no variants: - -```ts -export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2)); -checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } }); -``` - -Same-host consumers of a `local` type are usually reloaded together with their provider, but not always: another -plugin on the same host may be built against an older version. The rules are identical; only the variant argument is -missing. - -## Implementing a contract - -`Implementation` is the type of everything a provider must supply for the current version: one state -getter per member and variant, and one asynchronous method per method, each receiving `(args, ctx)`: - -```ts -const impl: Implementation = { - state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } }, - methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) }, -}; -``` - -Keep declaration types **inferred** (no annotation on `NotesV2`): the inferred type carries the complete history, and -`checkAccepts` needs it to know what changed. - -:::review | Review: L1's implementation shape and the factory's return value -`Implementation` in `src/next/contracts/types.ts` is `{ state: { variant: { member: getter } }, methods }`. The -design's factories (`design.md` §6, used throughout this guide) return `{ variant, member: value, method: fn }`. -The objects layer (L4) has to connect the two, and also define how `serve` receives the checked accepts (this guide -uses the working name `{ accepts }` as a third argument) and what `"refuse"` does at runtime. The natural reading, -used in the stepper above, is that a refused method call and a hold that needs a refused state member end with -`contract_mismatch`; no source says so. -::: - -## The fingerprint - -The fingerprint is a 64-bit hash (`hash64`, portable across runtimes) of a **normal form** of the contract's -schemas. The normal form decides what counts as a change: - -| Ignored | Significant | -| --- | --- | -| order of object keys, `required` and `enum` (sorted) | order of union and intersection branches (`anyOf`, `oneOf`, `allOf`) and of tuples | -| annotations on schema nodes: `title`, `description`, `examples`, `$comment`, `deprecated` | defaults, formats, patterns, bounds, `$ref` names | -| placement | input, variants, member kinds and every member schema | - -Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. -Annotations are dropped only on schema nodes; an object property that happens to be named `title` is data and counts. - -Only data schemas are allowed. One walker copies, validates and normalizes each schema against a **whitelist of -TypeBox keywords**: primitives, arrays and tuples, objects and records, unions and intersections, `$defs` and `$ref` -(use `$ref` for recursive types), constraints and annotations. Conditional and dependent schemas, `not`, -`definitions`, `contentSchema`, `unevaluated*`, `contains`, `propertyNames`, unknown keywords, executable or custom -TypeBox features, arbitrary `Unsafe` and refinements, accessors, symbols and cyclic objects are rejected at -declaration. Bytes are the one special leaf: `bytes({ min, max })` checks a `Uint8Array` and has the data marker -`{ type: "bytes", minLength?, maxLength? }`, so it hashes like any schema; TypeBox's checking still works on it. - -Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. -`NORMAL_FORM_VERSION` is part of every digest, so a change of the normal form is a link protocol change. - -**Not a security boundary.** A fingerprint says "we were compiled against the same contract"; it proves nothing. A -hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own -schemas (chapter IV), whatever the fingerprint says. - -## Rules of thumb - -- Treat a published contract like a published API: every change is `evolve`, every `evolve` needs a decision about - old consumers. -- Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over - the value, not a query. -- Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the - metrics show no old bindings. -- Change the input or the variants only with a new base, and only when updating every consumer is acceptable. -- Contract, document and job versions are separate and each is decided on its own. Pinned notes needed `NotesV2` - for the wire and `NoteList` version 2 with a `migrate` that adds `pinned: false` (chapter V), but no new - `ImportJob` version, because its outcome did not change. - -:::note | Built so far -This chapter's contract layer is built: `packages/chord/src/next/contracts/` (commit `7d2bb8ea8`, about 1,300 lines -with `remote`, `local`, `method`, `evolve`, `checkAccepts`, `Implementation`, the schema helpers, `fingerprintOf`, -`schemaNormalForm` and `NORMAL_FORM_VERSION`), with nine test files covering evolution, fingerprints, the hash, -local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no -mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to -the objects and protocol layers, not built yet. -::: - -# On the Wire {#wire lede="What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces."} - -Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord -moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when -you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and -no callbacks: each of those rules removes a cost on some hop. - -## What a state member sends - -Every exposed document, derive or readable is a **state member**. Over a hold it produces one of these: - -| Update | When | -| --- | --- | -| `snap` | on hold and on re-hold: the whole value, once | -| `ops` | per commit: the draft's recorded ops, no diff | -| `ops` from a diff | per change of a derive (and of an adapted binding, chapter XVI) | -| throttled `ops` | job progress | -| `reset` | a migration commit replaced the value (adapted for older consumers) | -| `error` | the member failed | - -**Commits are atomic for consumers.** On the same host, subscribers are notified once, after every document of the -commit is adopted. Remotely, one `state` message per hold carries a batch of member updates that is applied before -listeners run, and a commit that touches several holds on one link travels as **one** `state` message on that link. -A consumer never sees half a commit. - -## Messages - -A message is a **header plus opaque parts** (R1). There is one part per hold the message concerns; the header says -which hold each part belongs to and how large it is. - -| Direction | Kind | Parts | -| --- | --- | --- | -| consumer to provider | `hold`, `release`, `call`, `cancel` | `call` has one (the arguments) | -| provider to consumer | `bound` | one: the snapshot | -| | `state` | one per hold the commit touched on this link | -| | `result` | one | -| | `gone` | none | - -There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link. - -**One binary format on every link (Q4):** `u32 header length | header JSON | part bytes`, as one `Uint8Array`. A -WebSocket sends it as a binary frame, IPC passes it with `serialization: "advanced"`, a memory link delivers it -asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol -layer; link adapters only move bytes. Tests on memory links therefore run the production path. - -A part is encoded **once, at its origin**, with that hold's own Delta encoder (paths are interned per hold, end to -end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, -whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec. - -**The header is a TypeBox schema (R4)**, a union tagged by `kind`; the protocol types derive from it. Headers from -untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and -workers) are not checked. **Unknown ids are ignored on both sides (Q1).** They are normal races: a `result` crossing a -`cancel`, a `call` crossing a `gone`, a state part crossing a `release`. The side that ended a hold or call settles -its own pending work: a consumer that receives `gone(h)` rejects its pending calls on `h`, and the order per -direction guarantees nothing else for `h` follows. Only a header that fails the schema is a protocol error, and it -closes that link. - -The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get -the server's protocol. - -## Through a relay - -A tab never talks to a worker directly (chapter I). The server **relays**, and a relay reads and rewrites only -headers: it maps hold and call ids between its two links, stamps the principal on `hold` and `call` (a client's -principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination -link, so each link gets only its own holds' parts. It never parses a payload. - -:::sequence animate | The owner adds a note while Ann watches. The commit leaves the worker as one `state` message with two parts; the server splits it by destination without decoding either part. -actor owner "owner's tab" "hold h1" blue -actor ann "Ann's tab" "hold h2" muted -actor server "server" "relay" accent -actor worker "worker S1" "NoteStore, Notes" -owner -> server: call c7 Notes.add (1 part) -server -> server: map ids, stamp owner -server => worker: call c3 (part forwarded as is) -worker -> worker: method runs, self.tx commits -worker => server: state: parts for h1', h2' -server -> server: group parts per link -server --> owner: state (h1 part) -server --> ann: state (h2 part) -worker --> server: result c3 -server --> owner: result c7 -::: - -The `state` message reaches the owner's tab before the `result`, so when `await notes.add(…)` resolves, the new note -is already in `notes.value`. Ann's view is a different variant (reader) and gets its own part from its own derive; the -relay neither knows nor cares. - -## Flow control - -Each link has **one byte window**, counted from each message's `byteLength` (parts are already encoded bytes; nothing -is measured or stringified to count it). The window covers every message, first snapshots included. - -- **Snapshots are always admitted (R19)**, even when larger than the window; they use it up. Whether a hold - **overflows** is decided on the changes queued after the snapshot. -- **Under backpressure** a commit's ops are collected per hold and encoded into one part when credits return. The - consumer still gets the commit atomically. -- **After an overflow** the hold is dropped and established again once the previous snapshot's debt has drained - (credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot - in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window. -- **Calls** are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. - Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation - protect them. -- **Byte-heavy traffic** such as terminals uses separate channels (chapter XIII), so a large state message never - holds up keystrokes. - -**Recovery:** after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in -flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and -re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the -per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything. - -## Size limits - -Limits apply **per part** (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the -header before anything is decoded; zlib's `maxOutputLength` enforces the decoded limit in the one decode function. - -| Situation | What fails | -| --- | --- | -| a part over the limit | only its own hold or call, with `too_large`; the other parts of the message apply | -| a snapshot over the limit | its hold ends `failed`, with no automatic re-hold | -| a WebSocket frame over `maxPayload` (e.g. 64 MiB) | the connection closes (code 1009); this is the transport's hard cap | - -Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The -decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission -(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own. - -Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP -component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII). - -## What each hop costs - -Chord v2's goal is as little compute and memory as possible while staying correct: **O(size of the change)** per -commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new -viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives -are almost free: `diffRevisions` on shared subtrees takes 0.018 ms, `applyImmutable` of a small change 0.004 ms. - -| Hop | Work | -| --- | --- | -| commit | Delta's `prepare` (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta | -| reads in a transaction | the draft overlay, no copy; an unchanged draft writes nothing | -| subscribers on the same host | the same value and ops, by reference | -| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, chapter VIII) | -| send | one encoding per hold part at its origin, size taken from the encoded bytes | -| relay | header only: ids mapped, principal stamped, parts grouped; parts never parsed | -| receive | compiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized | -| apply | `applyImmutable`, structure shared | -| new hold | the value encoded once (unavoidable) | -| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | - -These are targets. The frozen kernel measured far worse, which is why the design is so strict here: - -:::stats -- 35 ms | one small change to a 500 KB value in the frozen kernel, no viewer -- +13 ms | per viewer on top (10 viewers: 130 ms) -- 17 / 33 / 101 ms | a tiny SQLite commit after 0 / 10 / 50 earlier deltas -::: - -Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, -JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and -a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes -0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the -before/after per hop is in `apps/p2/docs/p3-reviews/v2-cost-model.md`. - -:::review | Review: v2-cost-model.md is partly outdated -The measurements and the per-hop "after" targets in `apps/p2/docs/p3-reviews/v2-cost-model.md` are current. Its -closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions -once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), -and §10 still describes the description fetch. -::: - -### Cost guards - -Chord reports **full-value walks, deep copies, encodes, schema validations and materializations** as ordinary -telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the -in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers -does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A -`vitest bench` suite measures the hops. - -### Ways to make it slow anyway - -| Mistake | Why it costs | Instead | -| --- | --- | --- | -| a derive that builds new objects for unchanged data | every commit diffs and sends O(value) | keep unchanged items identical: `mapShared`, `previous` (chapter VIII) | -| one long array that changes often (R24) | sharing is per path; each change copies one slot per element and visits each | keyed documents or a bounded window | -| bytes as base64 strings | inflated, parsed, compressed as text | `bytes()` leaves; chunked transfers for files | -| awaiting I/O inside `self.tx` | the object's line is blocked for every caller | do the I/O first, then a short transaction | -| storing registry entries or references in documents | stale after a reload, not data | look up the current value at use | -| opening natives at module level or in a facet | no owner: never closed, never restarted | `self.open` in an object (chapter IX) | -| callbacks as arguments | cannot cross a link, keep old code alive | registries, jobs, flows | -| one big object that everybody views | one large snapshot per viewer and per reconnect | split by viewer need; views of the same variant share one value | - -## Telemetry - -Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`. Each host gets **one adapter**, -`createHost({ telemetry })`, NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts -hold the adapter, to start root spans. Code sees spans: - -- `ctx.telemetry` is the current span (`Ctx` extends `Context`, so it is the same field Durable and pi-ai read). Every - span carries its adapter's metrics and propagation, so code with a `ctx` can record a metric or inject a carrier. -- `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child `ctx`. -- App values such as the principal stay out of Durable's context. - -**Across links**, `call` and `hold` carry the caller's trace carrier as a bounded header field (R5: limited key count -and string lengths); the provider runs the method under it, and each relay adds a `chord.relay` span. The call from the -sequence above, as a trace: - -```text -chord.call Notes.add [owner's tab] -└─ chord.relay [server] - └─ chord.method Notes.add [session:S1] principal=owner - · event chord.commit docs=2 ops=5 -``` - -Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from -another process are checked (keys present, strings), and a bad one falls back to a new root. - -**Shared work** (one activation serving several `get`s, a worker start) is its own root span, linked to the trigger -that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background -closes are root spans too, linked to their trigger where there is one. - -| Tier | What | -| --- | --- | -| spans | calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs | -| events | commits inside methods, retries, parking | -| metrics | state messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards | - -Ids are attributes of spans, never of metrics. **Nothing is measured or walked for telemetry**; only sizes the code -already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an -error. Chord ships a typed telemetry schema, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable -(`DURABLE_TELEMETRY_SCHEMA`) do. - -**Storing and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare) stores one -row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every -second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface `TelemetryStore` is shared: -`findTraces` (by name, attributes, errors, time window, with limit and cursor) returns complete traces, `getTrace` -one, `findMetrics` metrics. One renderer, `pi-telemetry/render`, prints traces as above; it prints links and never -follows them. The playground's trace viewer reads the same store. - -**Durable** already emits submit and phase spans, stores carriers on submissions and tasks (`TaskOptions.trace`), -and pi-ai emits `pi.ai.request` and `pi.ai.poll`. With the harness given the host's adapter (chapter XIV), an agent -tool that calls `Notes.add` through `chord(fn, context)` shows up as one tree from the model request down to the -commit. - -:::note | Built so far -Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in -Chord is L11. Built in pi: `pi-telemetry` (propagation, links, metrics, the store interface, the renderer), -`chord/context` with `Context.telemetry`, pi-ai and Durable telemetry, and `chord/telemetry-sqlite`. Delta is built; -`mapShared` and bytes as atomic leaves in `diffRevisions` are still to do. The frozen kernel sends descriptions on the -wire and closes a link on an unknown id; both are replaced. -::: - -# Building p2: the Core {#p2-core lede="From a list of what the app must do to a set of owners: the base code every p3 plugin builds on."} - -The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the -next two assemble **p3**, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in -`apps/p3`, fully isolated with its own data directory, and **without migrating p2's data** (no import of -`catalog.sqlite`, `shares.sqlite` or the subagents table). Its plugins are derived from what the app must do, not -ported one to one from p2's plugin layout. - -:::review | Review: p3 is not designed yet -The design fixes the rules (`design.md` §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the -per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, -derived from the functionality map in `apps/p2/docs/p3-functions/`. Object names are working names. Where a domain -has an open decision, the text says which. -::: - -## Start from functions, not from plugins - -The functionality map in `apps/p2/docs/p3-functions/` lists what p2 does, in plain words, with four facts per -function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups -the functions into 22 domains: - -| # | Domain | # | Domain | -| --- | --- | --- | --- | -| 1 | sessions | 12 | access | -| 2 | session workers | 13 | models, credentials, settings | -| 3 | conversations and agent | 14 | prompt resources | -| 4 | shell commands | 15 | status and presence | -| 5 | chat mode | 16 | search | -| 6 | agents (subagents, one-shots, peer messages) | 17 | remote access and forwards | -| 7 | code mode | 18 | logs and notices | -| 8 | session files | 19 | dev and reload | -| 9 | environments and execution | 20 | web shell | -| 10 | workspace | 21 | TUI shell | -| 11 | terminals | 22 | transport | - -The four facts answer the four questions every Chord object needs, in order: - -:::flow wide animate | From a function in the map to code. Each fact in the map answers one question Chord needs answered. -row "The map says" -box data "data" "what, how big" -box dur "durability" "ephemeral · until restart · persisted · transcript-bound" -box who "who" "owner · guest read · guest write · worker" -box where "where" "server · worker · web · TUI · environment" -row "Chord needs" accent -box owner "the owner" "smallest object it belongs to" accent -box store "where it is kept" "Durable, persisted or in-memory document, tab" -box variant "variants" "and revocation" blue -box host "host and code" "placement; core or plugin" -arrow data -> owner -arrow dur -> store -arrow who -> variant -arrow where -> host -note "Durability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with a stable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client." -::: - -**One owner per thing, the smallest object it belongs to.** A guest's grant belongs to the access owner, not to the -session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and -rules set at creation are written in the creating commit (Durable's `createConversation({ init })`). - -**Core or plugin.** Core owners are what the app always needs and everything builds on; they are always present, so -other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature -does so through a registry it owns itself (chapter XIII). The risk to avoid is a **core god object**: each core owner -owns only what belongs to it. - -## The ownership map - -The result of that procedure for p3, as a starting point: - -| Domain | Owner object(s) | Host | Code | -| --- | --- | --- | --- | -| access | `Access` (grants, devices, share links), `authenticate` | server | core | -| sessions | `Catalog` (session records), lifecycle jobs | server | core | -| session workers | the supervisor (`route(Session)` + idle stop) | server | core (app code) | -| session location | `SessionLocation { session }` | worker | core | -| conversations and agent | `Harness`, `ConversationStore`, `Conversation` views | worker | core | -| status and presence | `SessionStatus` (server copy), reporters on workers | both | core | -| shell commands | `ShellJob` in the chat plugin | worker | plugin | -| chat mode | chat plugin | worker | plugin | -| agents | subagents plugin (Durable child conversations) | worker | plugin | -| code mode | code mode plugin (sandbox native) | worker | plugin | -| session files | `SessionFiles { session }` (proposed, #17) | server | core or plugin, open | -| environments | envs plugin (config, SSH pools #20) | server and worker | plugin | -| workspace | workspace plugin | worker | plugin | -| terminals | terminal host process (#23), terminals plugin | separate process | plugin | -| models, credentials | models plugin, login job + flow | server | plugin | -| settings | `Settings` (single writer, #21) | server | core | -| prompt resources | prompt plugin (watchers in the session's environment) | worker | plugin | -| search | search plugin (index on the server, feed on workers) | both | plugin | -| remote access, forwards | radius plugin, ports plugin | server | plugin | -| logs and notices | `Notices` (one channel), log capture | server | core | -| dev and reload | the reload control, `reload_plugins` tool | server | plugin | -| web and TUI shells | UI facets over the objects above | clients | shells | - -The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins. - -## The base code - -```text -apps/p3/ - src/ - hosts.ts Server, Session host kinds - principal.ts P, Revocation, principalKey, revokedBy, can() - server/main.ts createHost, load, route(Session), WebSocket and Radius entry points, supervisor - worker/main.ts createHost with the start data, connect to the parent - core/ the core facets: Access, Catalog, SessionStatus, Settings, Notices (server); - SessionInfo, SessionLocation, Harness, ConversationStore (worker) - plugins/ chat, subagents, codemode, envs, workspace, terminals, models, login, - prompt, search, ports, radius, files, dev -``` - -`hosts.ts`, `principal.ts`, the `SessionInfo` start data and the `Harness` object are the ones the guide already -built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in -`plugins/` reloads. - -## Who is calling: authentication - -Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by -`authenticate(req)` when a WebSocket connects (chapter III): - -| Way in | Principal | -| --- | --- | -| loopback without a cookie (TUI, CLI, first page load) | `{ kind: "owner", access: "local" }` | -| loopback owner browser cookie (a per-port secret file) | `{ kind: "owner", access: "local" }` | -| a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token | `{ kind: "owner", access: "device", device }` | -| a guest cookie from a share link | `{ kind: "guest", session, mode, grant, revision }` | -| the Radius relay channel | `{ kind: "owner", access: "relay" }` | - -The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from -another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, -single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket -before Chord sees it. - -**One permission matrix.** Every function in the map names who may use it, and p2 checks those rules in several -places. p3 writes them once, in `principal.ts`, and every factory and `authorize` uses it: - -```ts -// src/principal.ts -type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage"; - -export function can(p: P, action: Action, session?: string): boolean { - if (p.kind === "owner" || p.kind === "server") return true; - if (p.kind === "worker") return action !== "manage" && p.session === session; - if (p.kind !== "guest" || p.session !== session) return false; - if (action === "terminal" || action === "manage") return false; // terminals: owner only - if (p.mode === "read") return action === "read"; // no workspace for guest read - return true; // guest write: send, upload, stop, workspace -} -``` - -Factories pick variants from it (`can(p, "send", session) ? "writer" : "reader"`), so the matrix is visible in one -file and the members a caller lacks are simply not there. - -## Access: grants, devices, share links - -`Access` is the core owner of everything that lets someone in: share links per session and mode, guest records, the -device list and the signing key. It lives on the server as the chapter IV pattern: a `local` store that persists the -documents (only stable identities may), and a `remote` view for the owner's settings screen: - -```ts -// src/core/access.ts (server, in the core server facet) -const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true, - initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 }); -const Guests = defineState("access.guests", { … }); // same shape: map(Guest), persisted -const Devices = defineState("access.devices", { … }); // map(Device), persisted - -host.serve(AccessStore, (_input, self) => { // local: may persist, acts for the host - const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices); - return { - links, guests, devices, - removeGuest: async ({ grant }, ctx) => { - await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx); - host.revoke({ grant }); // live access ends now - }, - resetLink: async ({ session, mode }, ctx) => { - const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx); - for (const grant of blocked) host.revoke({ grant }); - }, - signOut: async ({ device }, ctx) => { - await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx); - host.revoke({ device }); - }, - … // shareLink, join, signOutAll - }; -}); - -host.serve(Access, (_input, principal, self) => { // remote view: owner only - if (!can(principal, "manage")) throw forbidden(); - const store = self.get(AccessStore, {}); - return { - variant: "owner", - guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)), // names, modes; no tokens - removeGuest: (args, ctx) => store.removeGuest(args, ctx), - resetLink: (args, ctx) => store.resetLink(args, ctx), - signOut: (args, ctx) => store.signOut(args, ctx), - }; -}); -``` - -`authenticate` is base code on the same host and checks `AccessStore`'s current documents on every new connection -(it holds a reference to the store, taken with `server.root` at start). Live access and lasting denial stay separate, as chapter VII explained: - -| Event | Lasting effect (data) | Live effect | -| --- | --- | --- | -| remove a guest | guest record blocked | `revoke({ grant })` | -| reset a share link | new token; its guests' grants blocked | `revoke({ grant })` for each | -| archive a session | grants of that session void while archived | `revoke({ session })` | -| restore a session | grants valid again; revision bumped | none: guests reconnect | -| sign out a device | device revoked until its expiry | `revoke({ device })` | -| sign out all devices | signing key rotated, tokens cleared | `revoke({ device })` for each | - -A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the -block is in `Guests` and `authenticate` refuses it. Names are unverified, cleaned, at most 40 characters, unique per -session ("Ann (2)"). - -## The catalog - -`Catalog` owns the session records: id, title (and whether it was edited), location, model default, pinned, -archived, timestamps. It lives on the server, persisted, and it **never starts a worker**: listing sessions, showing -their status or their unread marks reads server-side copies only. - -```ts -export const Catalog = remote("p3.catalog", { - version: 1, - input: obj({}), - placement: () => Server(), - variants: { - owner: { - state: { sessions: obj({ items: arr(SessionRow) }) }, - methods: { - create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs), - archive: method(obj({ id: str() })), restore: method(obj({ id: str() })), - }, - }, - guest: { state: { sessions: obj({ items: arr(SessionRow) }) } }, // only the guest's own session, if not archived - }, -}); -``` - -Behind it sits a persisted `local` `CatalogStore`; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, -relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: **the catalog does not know which -plugins exist.** Archiving a session writes `archived: true` and revokes the session; every plugin that keeps -per-session things reconciles from `catalog.sessions` with a routine (chapter X), as the notes plugin did. The ports -plugin closes forwards of archived sessions that way, also after a crash. - -## On the worker: start data, location, harness - -Three core objects exist on every worker from its start: - -- **`SessionInfo`** (chapter IX): the start message as `constant(init)`, with the session id, data directory and the - location the server chose at start. -- **`SessionLocation { session }`** (working name), lifetime host: the current cwd and environment, with a location - revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it - with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core - per-conversation policy document. -- **`Harness`** (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened - under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root - model is not configured, and resumes interrupted Durable work on start. - -`ConversationStore` and the `Conversation` views in front of it are chapter XIX. - -## Workers: start, keep alive, stop - -Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is -`server.route(Session, { authorize, resolve })` plus a small server object that keeps per-session worker state: - -:::steps | A worker's life, as the supervisor sees it -+++ Authorize, then start -A tab gets `Notes { S1 }`. `authorize` checks `can(p, "read", "S1")` and that the session exists and is not archived -(an archived session refuses, except an index rebuild that starts the worker without resuming work). `resolve` forks -`worker/main.ts` with the start data from the catalog, bounded by a 60 s readiness timeout. -+++ A start failure sticks -If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and -every `get` fails at once with it, until the owner presses Retry. That clears the error and the related notices. -Guests see that the session is unavailable, not the error text. -+++ Counting -The core facet on the worker reports `host.activity` (`{ references, calls, jobs, lingering }`) and the harness's -running Durable task count to the supervisor object, from a routine whenever either changes. The call carries -`ctx.incarnation`, so a report from an older worker incarnation is ignored (chapter X). -+++ Idle stop -When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: -the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted -and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is -a job, so it counts. -+++ Crash -If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail -(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call -`resolve` again with backoff; a passive viewer recovers without a click. -+++ Shutdown -On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains -persisted. -::: - -## Status without a worker - -The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, -and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a -routine over the harness's state) calls `SessionStatus.report(summary)` on the server; `SessionStatus` keeps the latest -value per session and ignores calls whose `ctx.incarnation` is older than the newest it has seen; clients read -`SessionStatus`, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the -same pattern (chapter XIX). - -## Settings and notices - -**Settings** (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have -**one writer**: a core `Settings` object on the server owns the file, validates writes and serializes them; every -other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches -settings, models and auth in every process and writes them from two, which is decision #21. - -**Notices** are one channel: one core `Notices` object on the server, written by everything that fails or finishes -(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most -50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing. - -:::review | Review: settings and file formats (#21, #22) -Single writer for settings, models and auth (#21) and versioning on-disk formats such as `envs.json` like documents -(#22) are open p3 decisions in `design.md` §26. The construction above assumes "one owner on the server, others -follow its state" for #21. -::: - -:::note | Built so far -Nothing of p3 exists. The functionality map is complete (`apps/p2/docs/p3-functions/`, five area files plus the -condensed `README.md`), and the design's app rules (`design.md` §25) are decided. p3 starts after the layered rebuild -and the playground's first real proof. -::: - -# Sessions and Conversations {#p2-sessions lede="The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses."} - -This chapter builds the domains a person touches most: the session list and its operations, the conversation view -and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section -names the owner, the host, and the Chord parts it uses; code appears only where it shows something new. - -## The session lifecycle - -The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs: - -| Operation | Owner and form | Notes | -| --- | --- | --- | -| create | `Catalog.create`, a method | absolute directory, `~` expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used | -| rename, pin | methods on `Catalog` | title 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically | -| automatic title | a routine on the search feed | the first user message, line breaks to spaces, 100 characters | -| agent rename | an agent tool on the worker calling `Catalog` | root conversation only | -| archive, restore | methods; revocation; reconciling routines | below | -| clone, fork, relocate | jobs on the catalog store | outlive the call, cancellable, one outcome | -| delete | a job; decision #18 | below | - -### Archive and restore - -Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while -keeping its storage. The catalog does none of the plugin-specific parts: - -:::steps | Archiving S1 -+++ One write -`Catalog.archive({ id: "S1" })` sets `archived: true` in the session record, in one transaction. -+++ Revoke the session -The catalog calls `host.revoke({ session: "S1" })`. Every guest principal of S1 has `{ session: "S1" }` in its -`revokedBy`, so their objects end and their links close on the server and on the worker (chapter VII). New -connections are refused by `authenticate`, because grants of an archived session are void. -+++ Stop the worker -The supervisor reconciles from `catalog.sessions` with a routine: an archived session's worker is stopped, and -`authorize` refuses to start it again. -+++ Plugins reconcile -The ports plugin's routine on `catalog.sessions` closes forwards of archived sessions. The search plugin marks the -session's rows archived. If the server crashed between the write and these steps, the routines run again with the -current value at the next start: nothing is a notification that can be lost. -+++ Restore -`Catalog.restore` clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards -stay closed. -::: - -### Clone, fork, relocate - -Each is work that outlives its call, can be cancelled and has one outcome, so each is a **job** (chapter XI) on the -catalog's local store, keyed by the owner's request id. They differ in what they pause: - -- **Clone** copies the session's store, uploads, downloads and artifacts under a new id, with the source worker - paused (refused while work runs) and title "(copy)"; directories `0700`, files `0600`, no links, no overwrite; a - partial copy is removed on failure. -- **Fork** before a user message copies the active context as of the entry before it into a new session (title - "(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, - the new one is seeded empty. A failure archives the copy. -- **Relocate** (cwd, environment or both): refused while the session is busy (naming the running subagents) or when - a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates - the record, and moves the running worker in place by updating `SessionLocation`: natives that follow the location - switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable - commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence. - -:::review | Review: jobs as objects (#9) -The design defers whether a long relocation or a login is a record inside its owner (watched through `running`) or an -object of its own (`Relocation { id }`). The construction above uses records. Decide when login is ported -(`design.md` §15, §26). -::: - -### Delete - -p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session -(`store.retireByTag(SessionTag, id)`, chapter V), the worker's databases (the Durable store), the session's files, the -search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to -this outcome (`design.md` §25); how plugins take part is open. - -:::review | Review: session delete (#18) -Options in `design.md` §26: per-session objects plus `SessionTag` plus one session directory (delete is retire by tag -and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: -whether the search index is per session (deleted with it) or global with a hook. Who holds the store for -`retireByTag` is not specified either. -::: - -## Status, children, unread: reports to the server - -The session list must show status, subagents and unread marks for sessions whose workers are not running, and must -not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the -current summary, the server object keeps the latest value per session (persisted where it must survive a restart), -ignores calls from an older `ctx.incarnation`, and clients read the server object. - -| Summary | Source on the worker | Kept on the server | -| --- | --- | --- | -| status: running, error, task count, busy conversations, usage and cost | the harness's task graph and usage | `SessionStatus`, in memory: a stopped worker is "not running" | -| children: name, model, thinking, tools, cwd, last entry, archived | the subagents plugin's child list | `Children`, persisted | -| latest root answer | the search feed | `Unread`: compared with the owner's read mark, persisted, shared by all owner devices | - -Guests see their own session's status without the error text. The owner's read mark is written when the root -conversation is visible, and a new session starts as read. - -## Conversations - -The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older -pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them: - -- **`ConversationStore { conversation }`** (local, on the worker): opens the Durable watch as its native, reaching - the harness through the core-private `HarnessRuntime` registry entry, and exposes a **window** as a derive: the - newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays - put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list - (chapter VIII's window pattern). -- **`Conversation { session, conversation }`** (remote, placement `Session`): the per-principal view. Variants by - `can()`: `owner` (everything, including model, thinking and handoff), `writer` (guest write: send, steer, stop, - compact, shell), `reader` (the window only). - -```ts -host.serve(Conversation, ({ session, conversation }, principal, self) => { - const store = self.get(ConversationStore, { conversation }); - const policy = self.get(ConversationPolicy, { conversation }); // core: readOnly, followsSessionCwd - const author = authorOf(principal); // structured: kind, id, display name - if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page }; - return { - variant: can(principal, "manage") ? "owner" : "writer", - view: store.window, - page: store.page, // older history: returns data - send: ({ text, attachments, mode, requestId }, ctx) => { - if (policy.value.readOnly) throw forbidden("read-only conversation"); - return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx); - }, - stop: (_args, ctx) => store.stop(author, ctx), // allowed even when read-only - … - }; -}); -``` - -What each function becomes: - -| Function | How | -| --- | --- | -| send, retried send | Durable `submit` with the author's `requestId`: a retry with the same id is admitted once (Durable dedupes, chapter XIV) | -| steer or queue while busy | `whenBusy: "steer"` or a follow-up; the inbox is `docs["pi.inbox"]` in the window | -| attribution | the view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text | -| wait for prompt resources | the first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows) | -| stop, dequeue | withdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their **restored drafts** | -| restored drafts | a keyed document per (conversation, author), tagged `SessionTag`, persisted in the store; claimed once, first tab wins | -| read-only conversations | the core per-conversation policy document, written at `createConversation({ init })` by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed | -| history pages | a method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included | -| model and thinking | owner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default | -| compaction, handoff | Durable operations through the store | - -**Drafts in the composer** are UI state, not Chord state: the web shell keeps one draft in `sessionStorage`, the TUI -keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged -("Sending when connected"). Only restored drafts, which the worker produces, are documents. - -## Shell commands - -`!cmd` runs a command in the conversation's cwd on its environment and records the result as an entry; `!!cmd` does -the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is -at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the -run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is: - -```ts -const ShellJob = defineJob("chat.shell", { - version: 1, - args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }), - progress: obj({ output: str() }), - result: ShellResult, - concurrency: 1, // one per conversation -}); - -const results = self.state(ShellResults); // keyed by run id: dedupe after completion -const shell = self.job(ShellJob, { - run: async ({ command, include, author }, ctx) => { - const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" }); - const env = await environmentFor(ctx); // the conversation's environment, chapter XX - const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) }, - (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); }); - const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted }; - await conversation.submit( - { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx); - return data; - }, - finish: async (done, tx) => { - (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error }; - }, -}); -``` - -The request id `shell:${ctx.jobId}` makes the entry idempotent: an attempt interrupted after the write cannot write -it twice. Cancel is `shell.abort(id)`, allowed for the command's author and the owner. A retried `runShell` with the -same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after -completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does -not depend on viewers. - -## Chat mode - -Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new -subagents start with it off. Because it must fork and rewind with the conversation, it is **transcript-bound**: a -Durable conversation document, not a Chord document. Messages without `@pi` become display-only Durable `write` -entries (author, text, attachments, reply quote); a message with `@pi` becomes an input with a digest of the chat -since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply -quote is derived on the worker from the entry id the client names: the client never supplies author or quote text. - -## Subagents and one-shot agents - -Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one -`subagent` tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and -each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a -restart sends nothing twice. Chord adds only three things: - -- the **child list** for the sidebar is reported to the server (`Children`, above), so it shows without a worker; -- a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in - the core **per-conversation policy document** when it is created, so chat and relocation enforce it without knowing - the subagents plugin (`design.md` §25 porting guideline); -- archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots. - -Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with -the session's model choice. - -## Messages between sessions - -The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and -calls a server object through `chord(fn, context)` (chapter XIV): - -```ts -const peerSend = defineTool({ - name: "session_send", - execute: (args, api, context) => chord(async (ctx) => { - const peers = await ctx.get(Peers, {}); // server, worker variant - await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx); - return { output: [{ type: "text", text: "sent" }] }; - }, context), -}); -``` - -`Peers` checks that the sender is a root conversation of a non-archived session, refuses sending to itself and -archived recipients, then gets the recipient's `Conversation` (which starts its worker) and submits with the -sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried -tool call delivers once. The sender's -incarnation is `ctx.incarnation`; a call from a replaced worker is refused. Only owner-initiated turns may use the -tool; guests never. - -## Search - -The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no -Durable internal inputs) for title, cwd and message search across sessions, active or archived. - -:::flow wide animate | The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query. -row "Worker S1" blue -box commit "Durable commit" "in any conversation" -box wake "wake" "a routine on the harness's commit signal" -box page "read from cursor" "200 entries per page" -row "Server" accent -box index "SearchIndex" "native: SQLite FTS" accent -box cursor "cursor per conversation" "moves after the whole range" -box query "query" "terms, scope, at most 41 hits" -arrow commit -> wake -arrow wake -> page -arrow page -> index "report(page)" -arrow index -> cursor -note "Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing." -::: - -The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash -the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting -an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, -snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" -forward and supplies the latest root answer for unread marks. - -## Presence - -Viewers per session are `host.references(Conversation)` on the worker, counted per client (chapter VIII), and -reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence -is not keep-alive: the supervisor uses `host.activity`, which also counts calls and jobs. - -:::note | Built so far -None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, -child conversations, background tasks) and the Chord layers L3 to L9 (not built). -::: - -# Machines, Files, Models {#p2-machines lede="Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3."} - -The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, -terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are -places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, -three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per -thing. - -## Environments - -The **envs plugin** owns the environment configuration: name, host, user, port, identity file, shell and its -variables, plus a managed ed25519 key per environment and its own `known_hosts` (kept apart from the user's, aliased -per environment name). - -| Function | Owner and form | -| --- | --- | -| list, add, edit, delete (refused while a session, archived too, uses it) | `Envs` on the server: a persisted document behind an owner-only view | -| validation (no leading `-`, no whitespace or control characters in host and user) | the view's methods, before the write | -| a hand-edited `envs.json` is picked up; an invalid file keeps the last valid set and shows an error | a native file watcher (poll fallback) in the envs store; one writer (#21) | -| environment revision: a hash of the config, part of its identity | a derive | -| connection test (connect, deploy the daemon, platform, latency, warnings) | a job; the result per revision in a document | -| unknown or changed host key: fingerprints, trust after confirmation, forget | a flow (chapter XII) on the server | -| browse directories (`~`, relative paths, at most 10k scanned, 1,000 shown) | a method with a **call-scoped** SSH session (chapter IX) or the pooled connection | -| a changed environment restarts its sessions' workers once idle | a routine on the envs store and the status reports | - -**Execution in the agent.** The worker's harness asks `HarnessEnvironments` for an environment per target (chapter -XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over -SSH through the `pi-env` daemon, deployed per platform and connecting at the first operation. A remote session never -falls back to local tools. The environment follows `SessionLocation`: after a move, the old connection closes once -the move has committed and watchers reopen at the new location. - -```ts -// envs/worker.ts -export default defineFacet((host, self) => { - const location = self.get(SessionLocation, { session }); - const pool = self.get(EnvPool, {}); // one owner of SSH connections (#20) - self.contribute(HarnessEnvironments, (target) => - target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd)); -}); -``` - -:::review | Review: connection pools (#20) -SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, -terminal host). `design.md` §26 asks for one owner with a pool or deliberate per-use connections. `EnvPool` above is -the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or -key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change -rarely. -::: - -## Workspace and file index - -The **workspace plugin** serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: -list a folder (at most 2,000 entries, `.git` hidden), read a file (1 MiB, binary detected), git status, diff, log and -single commits, literal text search, and all paths for name filtering. Every query is a **method returning data**, not -state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, -bumped by a native watcher of the tree, `HEAD`, the index and config (debounced 100 ms); viewers reload what they show -when it changes. - -Two rules live in one place each instead of three: - -- **Confinement:** path rules, a canonical root, no `.git`, the app's private data directory denied (also excluded - from search). One helper used by listing, reading, uploads and the file index. -- **Git hardening:** no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s - timeout, output caps. - -Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that -reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp -file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it. - -The **file index** for `@` completion belongs to the same owner, so completion consumes the index instead of scanning -on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per -second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until -sent; sending them is an ordinary `Conversation.send`. - -## Terminals - -Terminals must survive server and worker restarts, so their PTYs live in a **detached terminal host process**, one per -data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals -plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the -session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). -Terminals are **owner only**. - -Bytes do not go over object links. Attaching opens a **channel** (chapter XIII) with a single-use token (30 s): JSON -control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a -viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current -screen from a headless emulator with 10k lines of scrollback. - -:::review | Review: the terminal host process (#23) -The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals -end), or deliberately outside it with a "host runs older code" notice as in p2? Open in `design.md` §26. -::: - -## Session files - -Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's `send_file` tool, versioned -artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all -per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory -layouts for them. The proposal is one owner: - -```ts -export const SessionFiles = remote("p3.session-files", { - version: 1, - input: obj({ session: str() }), - placement: () => Server(), // readable without a worker - variants: { writer: { … }, reader: { … }, worker: { … } }, -}); -``` - -`SessionFiles { session }` on the server holds the index (per kind, readable without a worker), the chunked upload -and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied -into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a -sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types. - -:::review | Review: session files (#17) -Proposed in `design.md` §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and -512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote -environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents. -::: - -## Ports and Radius - -The **radius plugin** keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in -by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, -60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two -kinds of channels: - -```ts -// radius/server.ts (the entry point: one line, as in chapter I) -radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); -radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value)); -``` - -Object links become owner principals with `access: "relay"`: owner, but treated as remote (no localhost-only -powers). p2 had to fake a loopback request with a secret header for this; with `accept` that is gone. - -Forward channels go to `localhost:` (Host, Origin and Location rewritten; 502 when nothing listens; anonymous -visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, -`radius.forwards`, and the **ports plugin** contributes its forwards (at most 50, random ids, a public toggle that -needs confirmation, an agent tool `ports` for top-level owner-initiated conversations). Without the ports plugin the -table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The -ports plugin closes forwards of archived sessions with a routine on `catalog.sessions` (chapter XIX). - -:::review | Review: may a plugin call `accept`? -`design.md` §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If -radius is a plugin, its facet needs the host's `accept`, which no other part of this guide gives a plugin. The -alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified. -::: - -## Models and credentials - -The **models plugin** owns the model runtime (pi-ai's models with custom providers and the model cache) in a -host-lifetime object, and the credentials. It contributes the runtime to `HarnessModels` (chapter XIV), so after a -models reload the next request uses the new runtime while a running stream finishes on the old one. **Credentials -never go to clients**: the views expose providers, login methods and which type is signed in, never tokens. Every -secret is registered for redaction in one place, the log capture. - -Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), -logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are -settings (chapter XVIII). - -### Login - -A login is the guide's flow example (chapter XII): a **job** on the server that runs pi-ai's login, and a flow that -shows its steps to the starting owner. - -```ts -const LoginJob = defineJob("models.login", { - version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }), - concurrency: 1, // one login at a time on the server -}); - -const login = self.job(LoginJob, { - run: async ({ provider, type }, ctx) => { - await runtime.value.login(provider, type, { - signal: ctx.signal, // 10 minute deadline, cancel, leaving - prompt: (prompt) => steps.ask(prompt, ctx), // text, secret, select, manual_code - notify: (event) => steps.notice(event, ctx), // info, auth_url, device_code, progress - }); - return { ok: true }; - }, - finish: async (done, tx) => { /* record the outcome; refresh the model list */ }, -}); -``` - -`prompt` is a promise-based callback inside the work; the resolver stays in the job (`steps.ask` records the step in -the flow state and waits for `answer`), and nothing crosses an object boundary. On a relayed connection the device -code method is selected automatically. The flow is visible only to the owner principal that started it; others are -refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine -aborts the job. Codes, URLs and typed secrets are redacted from logs. - -## Prompt resources - -`AGENTS.md`, skills and templates are read in the session's environment, watched (poll fallback, last valid state), -and expanded on the worker before sending. The **prompt plugin** owns them per session on the worker, contributes the -system prompt section and skills to `HarnessExtensions` as readable contributions (a change recomposes), and exposes a -readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers -reopen at the new location. - -## Code mode - -Code mode is a plugin (chapter XIV): its `pi-codemode` sandbox is a native; scripts call tools as nested Durable calls -and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the -old sandbox for at most 30 s. The on/off switch is a setting. - -## Logs and notices - -**Log capture** tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, -redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB -each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. -**Notices** are the one channel from chapter XVIII; dismissal stores only notice ids. - -## Dev and reload - -The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; -the `reload_plugins` agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for -base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a -successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; -the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (`0700`) accepts development -commands without a network admin port. - -## The shells - -The web and TUI shells are UI facets over the objects above. Two rules matter for Chord: - -- **UI state survives navigation, reload and reconnect on purpose.** Composer drafts, pending sends with their request - ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect - re-holds and gets snapshots; nothing is lost that was not in flight. -- **Leaving is never derived from an unmount.** A plugin reload remounts components (chapter XV). Cancelling a login, - closing an overlay or giving up a lease is an explicit action or a dropped reference. - -The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or -credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a -confirmation. That is app code around Chord, not part of it. - -## Transport - -The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded -on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with -snapshots after a reconnect, and a protocol version that makes an outdated web page reload. - -:::note | Built so far -None of p3 exists. pi-ai's login API, pi-codemode and the `pi-env` daemon exist and are used by p2 today; the HTTP -component and channels are deferred until after the kernel's first real proof (chapter XIII). -::: - -# Other Ways to Do It {#compared lede="Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs."} - -Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is -**Cordis**, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework -since 2019, and its most visible user, **DeepSeek Harness** (`dsh`), DeepSeek's open-source agent harness, whose -motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: -who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model -reaches beyond one process. - -:::note | Sources and status -DeepSeek Harness at commit `687f282` (release 0.2.1-rc.2), which vendors Cordis as `@deepseek-ai/cordis` 4.0.5 (Cordis -RC7 with local changes, `vendor/README.md`): its Cordis tutorial (`docs/cordis-tutorial/`), `docs/architecture.md`, -`docs/subsystems/persistence.md`, `docs/subsystems/core.md` and `packages/boot/hmr/README.md`; the Cordis paper "A -Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 -is the design this guide describes. The comparison is between a working system and a design. -::: - -## Cordis in one example - -A Cordis plugin is a function (or an object with `apply`, or a `Service` subclass) that receives a **context**. It may -declare the services it needs with `inject`, and everything it does to the world goes through the context: - -```ts -export const name = "greeter"; -export const inject = ["tools"]; // activate only while a `tools` service exists - -export function apply(ctx: Context) { - ctx.tools.register(defineTool({ name: "greet", /* … */ })); // a registration is an effect: undone on unload - ctx.on("session/start", (s) => log(s.id)); // an effect-bound listener - ctx.effect(() => { // any other side effect, with its disposer - const timer = setInterval(work, 200); - return () => clearInterval(timer); - }); -} -``` - -Each plugin instance is a **fiber** with states `PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED` (or `FAILED`). -Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs -the disposers. The paper calls this **temporal composability**: a component can be removed at any time and reverses its -own effects. Its counterpart, **spatial composability**, is the dependency side: `inject` is live, so a plugin starts -when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The -paper's claim is that independently written components can therefore be combined and replaced without disturbing each -other. The next sections check how far that holds. - -**Services** are named capabilities on the context (`ctx.provide("x", value)` or `class X extends Service`). Names are -runtime strings in one flat namespace per application; TypeScript declaration merging makes `ctx.x` typecheck, but -nothing checks the service's shape at runtime, and optional dependencies are probed with `ctx.get("x")`. Scopes come -from the context tree: `ctx.extend()` derives a child context, `ctx.isolate(name)` gives one subtree its own binding of -a service, so two parts of an app can use two different `shell` providers. - -**Hot reload** follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and -loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin -that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and -mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount. - -## DeepSeek Harness on Cordis - -dsh applies "everything is a plugin" to an agent: the model adapters (`ctx.llm`), the tool registry (`ctx.tools`), the -session log (`ctx.sessions`), the agents (`ctx.agents`) and the agent loop itself (`ctx.agentLoop`) are plugins and -services. A running dsh is a plugin tree composed at boot from a **profile** (`web`, `headless`, `sdk`, `acp`) that -stacks **bundles** of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent -loop is a config patch. - -| Concern | How dsh does it | -| --- | --- | -| sessions | a durable, append-only event log per session (JSONL backend, single writer, `flush` as the durability barrier); model history is derived from the events | -| crash mid-turn | the log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic `turn/end { reason: "interrupted" }` | -| tools | `ctx.tools.register(defineTool(…))`, an effect of the registering plugin; calls go through pre/execute/post policy waterfalls | -| approval | closed by default; `allowed-once` is the only grant; plugin installation needs full access or per-call approval | -| reload | a source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs | -| processes | one plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams | -| distribution | a Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes | -| subagents | local subagents with their own durable sessions, and external ones | - -## What the guarantees are in practice - -Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's -framing. An earlier review for p2 came to the same result (`apps/p2/docs/p3-reviews/dsh-compare.md`). - -- **Reversal is cooperative and unordered.** `_unload` runs all of a fiber's top-level disposers concurrently - (`Promise.all` in `vendor/cordis/src/fiber.ts`); order holds only inside one effect, and the primer tells authors to - "keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A - disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose - cancel throws and warning that "the work may be orphaned" (`packages/jobs/jobs-local/README.md`). Reversal is as - good as every disposer, and nothing checks that a plugin registered all its effects. -- **Behaviour depends on registration order, and a reload changes it.** `emit`, `waterfall`, `serial` and `bail` - listeners run in registration order (`docs/cordis-primer.md`). Registering appends the listener (`push` in - `vendor/cordis/src/events.ts`, unless `prepend`), so when a plugin reloads, its listeners move to the end of the - list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits - now sees a different decision first. -- **Dependencies are strings, and every provider reload restarts its dependents.** A fiber's activation key is the - concatenation of its injected providers' fiber ids (`_refresh` in `fiber.ts`), so replacing a provider unloads and - reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every - consumer" (`AGENTS.md`): there is no versioned contract between a provider and its consumers. -- **Reloading a core service ends live work.** Unloading the agent provider "stops and drains every live handle it - made" (`packages/core/agent/README.md`), so running agents stop; reloading the session persistence backend under - live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" - (`.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md`). -- **Registrations leave a gap.** A reloading plugin's tools and other registry entries are removed with its old fiber - and added again by the new one; anything that looks them up in between does not find them. - -None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components -interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register -every effect, order nothing by registration, and hold no state that must outlive a reload. - -## Side by side - -| | Cordis and dsh | Chord v2 | -| --- | --- | --- | -| unit | a plugin instance (fiber) in one context tree | an object per (type, input, principal), placed on a host (chapters II, III) | -| side effects | recorded on the fiber with `ctx.effect` and friends; on unload the disposers run concurrently, each as good as its author made it | declared through `self` (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI) | -| dependencies | service names as strings in one flat namespace, via `inject`; a provider reload restarts every dependent | typed contracts via `get(Type, input)`; a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII) | -| ordering | listeners run in registration order, which changes when a plugin reloads | registries compose with a declared `compose` function; nothing depends on load order (chapter XIII) | -| where code runs | one process; remote UIs over RPC to the Host | a tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII) | -| who is calling | not a framework concept | principals stamped by links, variants per caller, revocation (chapters III, VII) | -| state | the plugin's own business; dsh's session log is a service | documents written in transactions, persisted, versioned and migrated (chapter V) | -| reload | unload the old instance, load the new one: in-memory state and running work of that plugin end | pure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV) | -| a running agent turn during reload | entries of changed plugins stop; unloading the agent provider stops its live agents | the harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV) | -| crash recovery of a turn | the turn is closed as `interrupted` | pi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV) | -| contracts and versions | TypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schema | contract versions with fingerprints and provider-side mappings for old consumers (chapter XVI) | -| status | in production in Koishi since 2019; dsh released | a design with the contract layer built | - -## What the comparison says - -**The core idea is the same.** "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are -two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. -Both reject the module-level timer and the stored callback for the same reason. - -**The difference is what survives a reload.** Cordis reloads at plugin granularity: the old instance unwinds and the -new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory -state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on -registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as -records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what -holds OS handles. That is more machinery: transactions, a store, versions on every document and contract. - -**The other difference is reach.** Cordis composes plugins inside one process, and dsh adds RPC to its own clients. -Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are -principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per -session and guests in other browsers, needs that; a single-process agent does not. - -:::review | Review: what a reload does to a running tool call in dsh -The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live -agents (`packages/boot/hmr/README.md`, `docs/subsystems/core.md`). They do not say what happens to a tool call in -flight when only the tool's plugin reloads; the comparison above does not claim either way. -::: - -# What's Next {.epilogue lede="Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other."} - -The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, -factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and -mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the -preface while the agent keeps running. This last chapter says how much of that exists and what is still open. - -## Status - -:::stats -- ~10k | lines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only -- 1 of 11 | layers of the rebuild started: L1 contracts, built and reviewed -- 6–7k | target size of Chord v2 without tests -::: - -- **The design is decided** except the points marked below. `packages/chord/docs/next/design.md` is the target; - `packages/chord/docs/next/decisions.md` is the log and wins where they differ. -- **The rebuild** goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and - garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and - derive; L3 documents and store; L4 objects; L5 references and `ctx`; L6 protocol and links; L7 relay, routes, - revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord. -- **Built:** the contract layer (`src/next/contracts/`, commit `7d2bb8ea8`, chapter XVI). In pi: Delta, - `pi-telemetry`, `chord/context`, `chord/telemetry-sqlite`, pi-ai and Durable telemetry, Durable's required - `checkpointWhen`. -- **Next, in order** (`apps/p2/docs/p3-handoff.md`): decide Q12 and Q15, which the L1 spec needs; write the spec for - the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in - `packages/chord/docs/next/layer-1.md`); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect. -- **In pi:** `mapShared` and bytes as atomic leaves in Delta's `diffRevisions`. -- **Then** the playground web shell (React, esbuild) with the reload control and a preview at `p3.mariozechner.at`, - and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and - channel components, then p3, domain by domain. - -## Deferred decisions - -Kept with stable numbers in `design.md` §26, each waiting for a concrete case: - -| # | Topic | When | Chapter | -| --- | --- | --- | --- | -| 9 | jobs as objects: a record inside the owner, or an object addressed by its id | when login is ported | XI, XIX | -| 11 | where a flow lives: a keyed document of its owner, or an object `LoginFlow { id }` | when login is ported | XII | -| 14 | project plugins: per-directory plugins with trust prompts and snapshots | own pass over `plugin-loading.md` | XV | -| 16 | a Durable Object profile | when a DO target becomes concrete | | -| 17 | session files: one `SessionFiles { session }` owner on the server (proposed) | p3 design | XX | -| 18 | session delete: retire by tag plus a session directory, or owner hooks | p3 design | XIX | -| 20 | connection pools: one owner of SSH connections per environment, or per-use connections | p3 design | XX | -| 21 | config files with a single writer: one owner on the server, others follow | p3 design | XVIII | -| 22 | file formats (`envs.json`, downloads, uploads) as versioned interfaces | p3 design | XX | -| 23 | the terminal host process: inside the revision or deliberately outside | p3 design | XX | - -## Open from review round 3 - -Decisions (`apps/p2/docs/p3-reviews/v2-review3-summary.md`; Q1 to Q10 are decided): - -| # | Problem | Proposal | -| --- | --- | --- | -| Q11 | a singleton that does not exist yet: `undefined` or `initial()`? | singletons read `initial()`; `undefined` only for absent keyed documents | -| Q12 | stripping arguments (R7) against "nothing is copied" | copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft) | -| Q13 | `ctx.incarnation` on relayed calls | relays stamp the origin host's incarnation, as they stamp the principal | -| Q14 | a writable holding functions exposed as a state member | only members with a JSON schema in the contract are exposable, checked at serve | -| Q15 | arguments of `local` methods: validated or not | apply R7 to `local` too, or keep `local` as the one exception | - -Cuts the owner may choose (each removes a primitive this guide teaches): - -| # | Cut | What it costs | -| --- | --- | --- | -| C3 | shrink jobs to a runner over documents (no versions, `accepts`, `blocked`, `persist`), or move jobs out of the kernel | job versioning, or `self.job` for core-only users | -| C4 | routines: `on` and `loop` only | `every`'s due time across reloads | -| C5 | merge `lifetime: "host"` objects into the facet | a plugin's always-on natives share one failure domain | -| C6 | remove `broken()` for opens and `retryOn` | nothing; reload and Retry still work | -| C7 | remove provider-side `linger` | ten lines of app code | -| C8 | remove `state: false` (method-only references) | a method-only consumer pays one snapshot | -| C9 | remove `previous` from derives | derives that used it move to `mapShared` | -| C10 | one `cell(initial)` for `constant` and `writable` | nothing | -| C11 | remove `ctx.trace` (use `traceIn(ctx, …)`) | nothing | -| C12 | remove call credits (the byte window covers calls) | nothing | -| C13 | two timeouts (`closeTimeout`, `readyTimeout`) instead of eight | per-thing overrides | -| C14 | remove the `handle.change` and `handle.retire` shorthands | nothing | -| C15 | remove `inVariant` and `expectVariant` (keep `RefIn`) | nothing | -| C16 | remove facet-owned mutable documents and their transaction exception | atomic writes across an object and its facet move to one owner | -| C17 | no harness-specific start-barrier exception | factories before the barrier cannot depend on the harness | - -## Where the sources disagree - -This guide follows the decision log where it is newer than the design, and the design where it is newer than -everything else. The places that need a fix in the sources: - -:::review | Review: drift in the written sources -- `design.md` §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to - C17. -- `design.md` §3 has a garbled sentence, "Values are method arguments are stripped". -- `apps/p2/docs/p3-handoff.md` §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and - R7 to R26 are open, and "resume R6 next". -- `kernel.md` and `step-*.md` describe the frozen code: host values, a development freeze, consumer-side linger, - JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target. -- `chord-next-walkthrough.md` lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed - by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome - `{ ok }` (now `{ kind }`), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), - and hooks using `host.root` (now `chord(fn, context)`, Q6). -- `apps/p2/docs/questions.md` uses `self.resource` and `replace` (both rejected) and `host.root` in hooks. -- `apps/p2/docs/p3-reviews/v2-cost-model.md`: the measurements are current, its "Decisions this needs" section is - not (chapter XVII). -- L1's `Implementation` shape differs from the factory return value used in the design (chapter XVI). -::: - -# API Reference {.appendix #api lede="Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built."} - -## Hosts - -| Call | Meaning | Chapter | -| --- | --- | --- | -| `createHost({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })` | a host; with an address it is addressable and its own code acts as `principal`; without, a client | I | -| `host.load([{ name, facet }])` | load facets by plugin name; base code passes its core facet the same way | I | -| `host.connect(link, { principal })` | the link to the parent (at most one) | I | -| `host.accept(link, { principal })` | a link from a client; every request on it acts as `principal` | I | -| `host.route(Kind, { authorize, resolve })` | how to reach hosts of a kind; `resolve` returns `{ link, principal }` | I | -| `host.revoke(r)` | end matching objects and links here and below | VII | -| `host.root(fn, { principal?, signal?, telemetry? })` | a call scope with a `ctx` for code outside objects | VI | -| `host.get(Type, input, { signal, state?, parkTimeout? })`, `host.hold(…)` | a reference from outside objects; `release()` ends it | VI | -| `host.serve(Type, factory, { lifetime?, linger? })` | serve an object type (in a facet) | II | -| `host.registry(Registry)` | the composed value of a registry, a readable | XIII | -| `host.references(Type)` | who holds references to objects of a type on this host (presence) | VIII | -| `host.activity` | `{ references, calls, jobs, lingering }`, for keep-alive | I | -| `host.close()` | end the host: objects, natives, links | I | - -## Inside a factory: `self` - -| Member | Meaning | Chapter | -| --- | --- | --- | -| `self.state(Definition)` | a document handle | V | -| `self.tx(async (tx) => …, ctx)` | a transaction on the object's line | V | -| `self.derive(name, [sources], fn)` | a derived readable | VIII | -| `self.get(Type, input)` | a reference held for the object's life | VI | -| `self.open(open, close, { closeTimeout? })` | a native; `.value` once open | IX | -| `self.routines.on(readable, fn)`, `.every(interval, fn)`, `.loop(fn)`, `.custom(…)` | standing behaviour | X | -| `self.job(Job, { run, finish? })` | owned work that outlives its call | XI | -| `self.contribute(Registry, value)` | a contribution (facets and `lifetime: "host"` objects only) | XIII | -| `self.writable(initial)` | a host-local readable of any in-process value, with `set` | XIII | -| `self.retryOn(readable)` | a trigger that retries a broken activation | VI | - -**Document handles:** `value`, `get(key)` (keyed), `keys`, `change(fn, ctx)` / `change(key, fn, ctx)` and `retire(ctx)` / `retire(key, ctx)` (shorthands for -one-document transactions). **Transactions:** `tx.draft(handle, key?, { tags? })`, `tx.retire(handle, key?)`, -`tx.start(job, args, { key? })`. **Jobs:** `job.start(args, ctx, { key? })`, `job.abort(id)`, `job.running`, -`run.outcome(ctx)`. - -## References and `ctx` - -| Member | Meaning | -| --- | --- | -| `ref.` | a state member (readable: `.value`, subscribe) or a method | -| `ref.state`, `ref.onState(fn)` | `loading`, `live`, `failed` (cause, `retryAt`), `released` | -| `ref.retry()` | retry a failed reference now | -| `ref.release()` | end a reference taken outside objects | -| `ref.members` | the variant's members | -| `ctx.signal`, `ctx.principal`, `ctx.incarnation`, `ctx.get` | cancellation, caller, calling host's incarnation, call-scoped references | -| `ctx.telemetry`, `ctx.trace(name, attributes, fn)` | the current span; a child span with a child `ctx` | -| `ctx.jobId`, `ctx.progress(value)` | in a job's `run` | - -## Definitions - -| Name | What it declares | Chapter | -| --- | --- | --- | -| `hostKind(name, input?)` | a kind of host and its address | I | -| `remote(name, { version, input, placement, variants })` | an object type reachable across links | II | -| `local(name, { version, input, members })` | an object type on its own host | IV | -| `method(args?, result?)` | a method in a base declaration | II | -| `evolve(prev, (e) => …)`, `checkAccepts(contract, accepts)` | the next contract version; the provider's mappings (**built**) | XVI | -| `Implementation` | the type of a provider for the current version (**built**) | XVI | -| `str num int bool nul lit oneOf union obj arr map opt bytes unknown`, `Static` | schema helpers (**built**) | II | -| `defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })` | a document | V | -| `defineTag(name)` | a tag; `store.retireByTag(tag, value)` | V | -| `defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })` | a job type | XI | -| `defineRegistry(name, { version, compose, accepts? })` | an extension point | XIII | -| `bindFacet

()` | base code's `defineFacet`, with factories' principals typed as the app's `P` (working name) | III | -| `defineFacet((host, self) => …)` | a plugin's setup for one kind of host; imported from base code (`@studio/core`) | II, III | -| `constant(value)` | a readable that never changes (start data) | IX | -| `broken(error)` | a failure that is not retried by the backoff | VI | -| `inVariant(ref, …)`, `expectVariant(ref, …)`, `RefIn<…>` | narrowing a reference by variant | III | -| `mapShared(fn)` | an array map that keeps unchanged items identical (planned in Delta) | VIII | -| `OutputBuffer`, `Progress` | bounded output and throttled progress (`@earendil-works/chord/output`) | XI | -| `memoryLinkPair()`, `ipcLink(child)`, `webSocketLink(ws)` | link adapters | I | -| `DocumentStorage`, `registerDocumentStorageConformance` | the storage port and its shared test suite | V | - -# Timeouts and Defaults {.appendix #defaults} - -| Timeout or default | Bounds | Default | Override | -| --- | --- | --- | --- | -| `closeTimeout` | a native's `close`, then reported as a possible leak | 30 s | per native | -| call drain | an ending object's own call handlers | = `closeTimeout` | per type | -| `readyTimeout` | activation; an awaited `get` | 60 s | per `get` | -| park deadline | parked calls (link drop, loading, rollout) | 60 s | per `get` (`parkTimeout`); per type | -| routine handover | a new routine waiting for the old handler on reload | 10 s | per routine | -| `finish` retry | backoff of a throwing job `finish` | 1 s to 60 s | per job | -| rollout install | one host installing a revision | 60 s | none | -| linger | an object kept alive after its last reference | per type | per type | -| backoff | retries of transient failures, per address | 1 s doubling to 60 s | none | -| eviction | keyed persisted documents idle in memory | 10 minutes | | -| client WebSocket limits | per part | 16 MiB compressed, 64 MiB decoded | per link kind | -| compression | JSON parts compressed at their origin | above about 16 KiB | | -| worker keep-alive (p2, app code) | idle time before the supervisor stops a worker | 5 minutes | app | - -# Versions {.appendix #versions} - -| Surface | Version | On change | Chapter | -| --- | --- | --- | --- | -| object type | `version` plus fingerprint | `evolve` and `checkAccepts`; otherwise `contract_mismatch` | XVI | -| document | `version` plus schema fingerprint | pure `migrate` on access; no bump fails `broken` | V | -| job | `version` | `accepts` on outcomes and `running`; `blocked` without | XI | -| registry | `version` | `accepts` on contributions; unmatched ones left out and reported | XIII | -| native | none | restarts on reload | IX | -| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | -| link protocol | protocol version | refused; hosts update together | XVII | -| UI components, Chord and app APIs | none yet | exposed as they are | | - -# Errors and States {.appendix #errors} - -`ChordError` has a stable `code` and a finer `reason`; on the wire it is `{ code, reason?, message, details? }` -(specified in `layer-1.md`, not built). - -| Code | Meaning | -| --- | --- | -| `invalid` | input, arguments or placement rejected (`invalid/input`, `invalid/args`, `invalid/placement`) | -| `contract_mismatch` | the consumer's version and fingerprint are neither current nor accepted | -| `forbidden` | the caller may not do this | -| `gone` | the object or hold ended | -| `not_ready` | not activated in time | -| `released` | the reference was released | -| `too_large` | a part exceeded its size limit | -| `unavailable` | could not be reached before the deadline (`unavailable/locked` and others) | -| `aborted` | cancelled by a signal | - -| Type | Values | -| --- | --- | -| `FailureCause` | `{ kind: "transient"; error }` (retried with backoff) or `{ kind: "broken"; error }` (retried only by a trigger) | -| reference state | `loading`, `live`, `failed` (cause, `retryAt`), `released` | -| job outcome | `{ kind: "done"; result }` or `{ kind: "failed"; error: "threw" \| "aborted" \| "lost" }` | -| job phase | `running`, `finishing`, `blocked` | -| rollout status of a provider | `updating` (calls park), rejected (fail at once), same revision (`contract_mismatch`) | -| host install report | `installed`, `installed_failed`, `rejected` | - -# Rejected Designs {.appendix #rejected} - -From `design.md` §31, grouped. Each was considered and is not coming back without a new reason. - -| Area | Rejected | -| --- | --- | -| reload | drain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; `restart_required`; host values and a development freeze | -| state | version fences without transactions; volatile documents; per-group lines; optimistic transactions; `tx.get`; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store | -| objects and references | resources replaced in place (`replace`, `deps`, stale handles); revoking proxies on `.value`; consumer-side linger; detached references counting toward presence; an `optional` flag on references; `opaque` members; callbacks as method arguments; generic call or request ids; a stream type in Chord | -| registries | registries created at runtime; entries resolved through mutable indirection; generic registry keys with a `conflict` flag; numeric middleware priorities; registry composition as a staging check | -| principals and placement | principal field patterns and re-stamping for revocation; `storeKey`; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete | -| wire | an outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app | -| process and tooling | a topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; `defaultProjectTrust: "always"`; ceremony (`native/` directories, `defineNative`, lints); weakening type checking | - -# Glossary {.appendix #glossary} - -**Adapted binding:** a consumer bound at an older accepted contract version, served through the provider's mappings. -**Base code:** the app's own code, imported statically; a change restarts the hosts. **Client:** a tab or TUI host, -not addressable. **Contract:** an object type's declaration (name, input, placement, members, version). -**Contribution:** a value a plugin adds to a registry. **`ctx`:** what every call receives. **Derive:** a value -computed from readables. **Document:** a named JSON value an object owns. **Facet:** a plugin's code for one kind of -host. **Factory:** the synchronous function that declares an object. **Fingerprint:** a hash of a contract's or -schema's normal form. **Flow:** input into running work, kept as state by its owner. **Host:** one participant of the -app, with an identity and an incarnation. **Incarnation:** the id a host gets on every start. **Input:** an object's -key among objects of its type; with the type and the caller's principal key it identifies the object. **Job:** owned work that outlives its call. **Link:** an ordered connection between -two hosts. **Local type:** an object type reachable only on its own host, without principals. **Native:** an OS-level -value opened with `self.open`. **Park:** a call waiting for a reference to become live again. **Placement:** the -function from input to host address. **Plugin:** a directory with a `package.json`, reloadable. **Principal:** who -calls, stamped by the link. **Principal key:** the string that identifies a principal in object identities. **Pure -object:** an object without natives; swapped in place on reload. **Readable:** a value plus its ordered changes. -**Reference:** what `get` returns. **Registry:** an owner's extension point. **Relay:** a host forwarding messages -between two links, rewriting only headers. **Reload:** swapping a plugin's code in running hosts. **Restart:** ending a -process. **Revision:** the complete code set the hosts should run. **Revocation:** ending live access for principals -matched by `revokedBy`. **Routine:** standing behaviour of an object. **Server:** the addressable root host. **Shared -store:** a local object that holds data once behind per-principal views. **Tag:** a typed label on documents, used to -retire them together. **Transaction:** an atomic set of document writes on the object's line. **Variant:** the set of -members a principal gets. **View:** a per-principal remote object in front of a shared store. **Worker:** a host the -server starts for one session. - -# Sources {.appendix #sources} - -| Source | Used for | Weight | -| --- | --- | --- | -| `packages/chord/docs/next/decisions.md` | every decision, review rounds 2 and 3 | wins where newer | -| `packages/chord/docs/next/design.md` | the target design, all chapters | authoritative otherwise | -| `packages/chord/src/next/contracts/` and its `README.md` | chapter XVI | built code | -| `packages/chord/docs/next/layer-1.md` | errors, inputs, argument checks | draft spec | -| `apps/p2/docs/p3-reviews/v2-review3-summary.md` | open Q11 to Q15, cuts C3 to C17 | current | -| `apps/p2/docs/p3-reviews/v2-cost-model.md` | measurements, per-hop costs | current except its last section | -| `apps/p2/docs/p3-functions/` | chapters XVIII to XX | current | -| `apps/p2/docs/p3-handoff.md` | status and order of work | §3 partly stale | -| `packages/durable/README.md` | the harness, submit, tasks, subagents | current | -| `chord-next-walkthrough.md`, `kernel.md`, `step-*.md`, `apps/p2/docs/questions.md` | background only | lag the decisions (see What's Next) | -| DeepSeek Harness at `687f282` (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512) | "Other Ways to Do It" | external, built systems | diff --git a/packages/chord/docs/next/explainer/index.html b/packages/chord/docs/next/explainer/index.html deleted file mode 100644 index e13c03572..000000000 --- a/packages/chord/docs/next/explainer/index.html +++ /dev/null @@ -1,5409 +0,0 @@ - - -Chord v2: a guide to building apps that reload - - - - -

-
-
A guide · Chord v2 target design · 2026-10-20
-

Chord v2
A Guide

-

How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running

-
11layers in the rebuild, L1 to L11
~1,300lines built so far (L1 contracts)
6–7ktarget size of Chord v2, without tests
0.004 msDelta's apply per small change
-

This guide teaches the designed API of Chord v2 (packages/chord/src/next, export @earendil-works/chord/next) -from packages/chord/docs/next/design.md and the newer decision log packages/chord/docs/next/decisions.md, which wins -where the two differ. Most of it is not built yet: a first kernel of about 10k lines is frozen as a reference and -Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (src/next/contracts/, commit -7d2bb8ea8). Code in this guide is therefore the target, not something you can run today. Every chapter says what is -built. API names are working names. Notes marked review are points that are undecided, or where the sources -disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start.

-
-
- - -
-

Before we begin

-

It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session -S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through bash. A colleague, -Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way -round. He fixes one line in the notes plugin on the laptop and presses Reload.

-

What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The -agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is -lost, and no process restarts.

-

The obvious ways to get there fail. Restarting the processes loses the turn in flight, the open terminal and every -connection; pi-durable would resume the turn, but the test run and the connections are gone. Hot-swapping modules -in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that -nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the -old code is never closed, and two versions of the plugin now write the same data without any order.

-

Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: every piece -of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only -declare what they need. Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, -watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only -what touches the operating system, and route calls between browser tabs, the server and per-session worker processes -without the plugin author writing any of that.

-

This guide teaches Chord v2 by building an app called Studio: a small version of p2 (Pi Mobile), with a server, -one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first -chapters grow one plugin, @studio/notes, until it uses every concept; the last three chapters assemble the rest of a -p2-like app from the same parts.

-
Review: what exists today

Nothing in this guide runs yet except the contract layer. packages/chord/src/next/contracts/ (L1 part 1: remote, -local, method, evolve, checkAccepts, the schema helpers) is built and reviewed. The rest of L1 (inputs, -argument checks, principals, errors) is a draft spec (packages/chord/docs/next/layer-1.md). The frozen kernel in -src/next/*.ts implements an older version of the design (host values, a development freeze, JSONL storage, -descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code.

-
-

Ten things to know

-
    -
  1. An app is a tree of hosts. One server, any number of workers it starts on demand, any number of clients (tabs, -TUIs) that connect to it. Clients reach workers only through the server, which relays.
    2. -
  2. Objects are the unit of everything. An object type is a contract (name, input, placement, members, version); an -object is one instance per input and caller. The first get creates it, later gets share it, the last release ends -it.
    3. -
  3. Factories declare, Chord activates. A factory is synchronous: it declares documents, references, derived -values, OS handles, routines and jobs through self, and returns the members. Chord opens, binds and starts them, -and undoes them in reverse.
    4. -
  4. Links stamp who is calling. A caller's data never says who it is; the link it arrived on does. Each caller gets -its own object and the variant of members it may use.
    5. -
  5. Shared data lives once, behind views. A local shared store holds the data; per-caller remote views expose -what each caller may see. Only stable identities persist documents.
    6. -
  6. State changes only in transactions. Documents are JSON values written in self.tx; commits are atomic for -every viewer and travel as small operations, never as whole values.
    7. -
  7. Failures look like disconnects. A reference survives link drops, restarts and failures: calls park or fail, -values stay readable, and Chord retries by itself. There are no request ids.
    8. -
  8. Everything long-lived has a declared form. Standing behaviour is a routine, work that outlives a call is a job, -input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer -or a callback.
    9. -
  9. Reload swaps pure objects in place and restarts native ones. Objects without OS handles get the new code with -their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole -host on its old code.
    10. -
  10. pi-durable runs inside a core object. The agent harness is a host-lifetime object on each worker; models, -environments and tools reach it through registries, and tools reach Chord through chord(fn, context).
    11. -
-

Words used throughout

-

A host is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical -identity and an incarnation that changes on every start. A link connects two hosts. The server is the one -addressable root host; a worker is a host the server starts for one session; a client is a tab or TUI, not -addressable. A plugin is a directory with a package.json; its code for one kind of host is a facet. Base -code is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while -plugins reload. An object type (or contract) describes objects; an object is one instance; its input is -its key among objects of that type, and its placement computes from the input on which host it lives. A factory builds an -object; self is what a factory declares through. A principal is who calls, stamped by the link; a -variant is the set of members a principal gets. A reference is what get returns. A readable is a value -plus its ordered changes. A document is a named JSON value an object owns, written in a transaction. A -derive is a value computed from readables. A native is an OS-level value (a socket, a watcher, a process) -opened with self.open; an object with natives is a native object, one without is a pure object. A -routine is standing behaviour, a job is owned work that outlives its call, a flow is input into running -work. A registry is an owner's extension point, filled with contributions. ctx is what every call -receives: cancellation, caller, call-scoped references. A reload swaps a plugin's code inside running hosts; a -restart ends a process; a revision names the complete code set the hosts run.

-
Reading the code in this guide

Imports are shown once per file and then omitted. obj, str, arr and friends are Chord's schema helpers (they -return ordinary TypeBox schemas). Helper functions that belong to the app (authenticate, watchDir, uuidv7) are -named and not shown. Where the design gives no name, the guide says it uses a working name.

-
-
- -
I

The Shape of an App

Which processes a Chord app has, how they find each other, and the start code that connects them.

-

Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, -and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does -none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare -deployment routes through the edge. This chapter writes Studio's start code: server.ts, worker.ts, the browser -entry, and a test setup.

-

Three roles

-

A Node app on Chord has one shape:

-
CLIENTS: NOT ADDRESSABLESERVER: EXACTLY ONEWORKERS: ONE PER SESSION, ON DEMANDbrowser tabowner, localbrowser tabguest AnnTUIownerserverServer()workerSession({ id: "S1" })workerSession({ id: "S2" })workerSession({ id: "S3" })WebSocketWebSocketWebSocketIPCIPCA worker for S3 does not exist until something asks for an object that lives there.
Figure 1. The three roles. Clients connect to the one server; the server starts workers and relays between them.
- - - - - - - - - - - - - - - - - - - - - - - - - - - -
RoleAddressableWho it is comes fromLifecycle
clientnothe authentication of its linkconnects by itself
serveryes, exactly oneits own declarationruns
workeryes, many, by a typed addressstamped by the server, its parentstarted on demand by the app
-

The server holds what all sessions share (the session list, access, settings) and starts workers. A worker -runs one session: its agent, its files, its terminal connections. Clients are browser tabs and TUIs. Nobody can -send a request to a client, so a client can only serve objects that nobody outside it needs; chapter IV names them -local types.

-

Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a -remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. -A detached process such as a terminal host could become another kind of child of the server.

-

Host kinds and addresses

-

Every addressable host has a typed address. The app declares its host kinds once, in base code:

-
// studio/src/hosts.ts
-import { hostKind, obj, str } from "@earendil-works/chord/next";
-
-export const Server = hostKind("server");
-export const Session = hostKind("session", obj({ id: str() }));
-
-Server();                  // the address of the server
-Session({ id: "S1" });     // the address of the worker of session S1
-

There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the -kinds. An object type names its host with a placement function from its input to an address (chapter II), so the -type checker knows that Session({ id: session }) needs a session field.

- -

A link is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory -pair for tests, IPC for a Node child process forked with serialization: "advanced", and WebSocket. All three carry -the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: -IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep.

-

Each link has a direction. A host has at most one link to its parent (connect) and any number of links from -below (accept for clients, route for workers). Requests arriving on a link act as the principal the link was given; -chapter III explains why that matters.

-

The server

-
// studio/src/server.ts
-import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next";
-import { fork } from "node:child_process";
-import { WebSocketServer } from "ws";
-import { Server, Session } from "./hosts.ts";
-import { coreServerFacet } from "./core/server.ts";
-import type { P } from "./principal.ts";
-
-const server = createHost<P>({
-  address: Server(),
-  principal: { kind: "server" },                 // who this host's own code acts as
-  storage: await openServerStorage(dataDir),     // persisted documents (chapter V); memory if omitted
-  onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); },
-});
-
-await server.load([
-  { name: "@studio/core", facet: coreServerFacet(config) },   // base code, under a fixed name
-  ...(await loadPlugins("server")),                           // [{ name, facet }] from the plugin loader
-]);
-
-const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 });
-wss.on("connection", (ws, req) => {
-  const principal = authenticate(req);           // app code: cookie, loopback, device token
-  if (!principal) return ws.close(4401);
-  server.accept(webSocketLink(ws), { principal });
-});
-
-server.route(Session, {
-  authorize: (p, { id }) =>
-    p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id),
-  resolve: async ({ id }, { signal }) => {
-    const principal: P = { kind: "worker", session: id };
-    const child = fork("worker.ts", [], { serialization: "advanced", signal });
-    child.send({ id, principal, init: await initFor(id, signal) });   // the app's own start message
-    return { link: ipcLink(child), principal };                       // Chord stamps `principal` on it
-  },
-});
-

Each call has one job:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CallMeaning
createHost<P>({ address?, principal?, ...options })with an address: an addressable host whose own code acts as principal; without: a client. P is the app's principal type
load([{ name, facet }])the facets this host runs, each with its plugin's package.json name; duplicates and invalid names are refused. Base code passes its core facet the same way
accept(link, { principal })a link from a client; every request on it acts as principal
route(Kind, { authorize, resolve })how to reach hosts of a kind. authorize(principal, address) runs before anything starts; resolve(address, { signal }) returns { link, principal }
connect(link, { principal })the link to the parent, at most one, replaced after a reconnect; principal says who the parent is
root(fn, { principal?, signal? })a call scope for code outside objects (a CLI command): gives fn a ctx (chapter VI)
-

Chord has no listen. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay -delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls -accept. Radius, for example, is one more line:

-
radius.onChannel((ws) =>
-  server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } }));
-

The worker

-
// studio/src/worker.ts
-const { id, principal, init } = await startMessage(process);       // app code: the first IPC message
-const worker = createHost<P>({
-  address: Session({ id }),
-  principal,                                                       // { kind: "worker", session: id }
-  storage: await openSessionStorage(init.dataDir, id),             // one storage per session
-});
-await worker.load([
-  { name: "@studio/core", facet: coreWorkerFacet(init) },          // serves the start data (chapter IX)
-  ...(await loadPlugins("worker")),
-]);
-worker.connect(ipcLink(process), { principal: { kind: "server" } });
-

A browser tab

-
// studio/web/main.tsx
-const client = createHost<P>();                                    // no address: a client
-await client.load(await loadPlugins("web"));
-client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } });
-const notes = await client.get(Notes, { session: "S1" });          // tab → server → worker S1
-

The first get, step by step

-

The tab asks for the notes of S1. No worker for S1 runs yet.

-
browser tabownerserverroute(Session)app codeworker S1placement: Session({ id: "S1" })hold Notes { session: "S1" }not mine: route(Session)authorize(owner, { id: "S1" })trueresolve({ id: "S1" })fork + start messagecreateHost, load, connect{ link, principal: worker S1 }hold (stamped: owner)placement check, run factorybound: snapshotbound: snapshot
Figure 2. The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link.
-

resolve runs once per address while its link lives. When the link drops (the worker crashed or the supervisor -stopped it) and references to that worker still exist, Chord calls resolve again by itself, with a per-address -backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes -even if the request that triggered it went away.

-

Incarnations make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link -handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold -(chapter VI).

-

Tests: everything in one process

-

The same code runs in a test with memory links. resolve builds the worker host in memory and returns one end of the -pair:

-
server.route(Session, {
-  authorize: () => true,
-  resolve: async ({ id }) => {
-    const [up, down] = memoryLinkPair();
-    const principal: P = { kind: "worker", session: id };
-    const worker = createHost<P>({ address: Session({ id }), principal });
-    await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]);
-    worker.connect(up, { principal: { kind: "server" } });
-    return { link: down, principal };
-  },
-});
-

Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path.

-

What stays app code

-

Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord -provides one readable for the last one: host.activity, with { references, calls, jobs, lingering } (viewers, calls -in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server -together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for -five minutes, as p2 does. Chord counts; it has no idle policy.

-

Host options and timeouts

-

createHost takes the options introduced later in this guide: storage (V), principalKey and revokedBy (III, -VII), telemetry (XVII) and onError. onError receives background errors and { kind: "storage_failed" }; every -app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor -restarts it.

-

Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: -readyTimeout (60 s: how long an activation and an awaited get wait) and the park deadline (60 s: how long a call -waits through a link drop or a restart before failing unavailable).

-
Built so far

Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has -working versions with the same shape (createHost, connect, accept, route, load), which the rebuild -simplifies.

-
-
- -
II

A First Object

A plugin, a contract, a factory, and a browser tab that calls it on another process.

-

Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can -add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent -tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord -there is one object for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it.

-

The plugin's files

-

A plugin is a directory with a package.json. Its name (npm style, globally unique and stable) is the plugin's -identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a -directory name or load order.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FileRuns onHolds
package.jsonread by the loader"name": "@studio/notes"
contract.tsevery host that imports itobject types: names, shapes, method signatures; no running code
worker.tsevery session workerthe worker facet
server.tsthe serverthe server facet (later: the download route)
web.tsxbrowser tabsthe web facet: the panel
-

The contract

-

An object type is a contract: a name, an input (the object's key among objects of this type), a placement -(on which host it lives, computed from the input), the members (state you can read and methods you can call) and a version.

-
// notes/contract.ts
-import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next";
-import { Session } from "@studio/core/hosts";
-
-export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() });
-export type Note = Static<typeof Note>;
-
-export const Notes = remote("notes", {
-  version: 1,
-  input: obj({ session: str() }),                          // the key: { session: "S1" }
-  placement: ({ session }) => Session({ id: session }),    // input → the host's address
-  variants: {
-    editor: {
-      state: { notes: obj({ items: arr(Note) }) },
-      methods: {
-        add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()),   // returns the new id
-        remove: method(obj({ id: str() })),                              // no result
-      },
-    },
-  },
-});
-

remote means objects of this type can be reached from any host, across links (chapter IV introduces local). The -schema helpers (obj, str, int, arr, map, opt, lit, oneOf, union, bool, num, nul, bytes, -unknown) return ordinary TypeBox schemas; short options such as { min, max } bound lengths or values. Schemas must -describe data: no callbacks, no class instances, no custom codecs.

-

method() has four forms: method() (no arguments, no result), method(args), method(args, result) and -method(undefined, result).

-

A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its -description and computes a fingerprint, a hash of its input, variants and members. Two hosts compare contracts by -version and fingerprint (chapter XVI). Placement is not part of the fingerprint.

-

Placement

-

The input says which object; placement says where it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server -routes by it. The provider runs the function too and refuses an object that is not its own with invalid/placement. -So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed -function, a Session placement on a type whose input has no session field does not compile.

-

Inputs are normalized before anything else: defaults are applied, unknown fields removed, and a canonical key -computed. { session: "S1" } and { session: "S1", extra: 1 } are the same object.

-

The facet and the factory

-

A facet is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the -host which object types it serves:

-
// notes/worker.ts
-import { defineState } from "@earendil-works/chord/next";
-import { defineFacet } from "@studio/core";          // Studio's defineFacet: principals typed as Studio's P (chapter III)
-import { Note, Notes } from "./contract.ts";
-
-const NoteList = defineState("notes.list", {      // a document (chapter V); in memory for now
-  schema: obj({ items: arr(Note) }),
-  version: 1,
-  initial: () => ({ items: [] }),
-  checkpointWhen: () => true,
-});
-
-export default defineFacet((host, self) => {
-  host.serve(Notes, ({ session }, principal, self) => {
-    const list = self.state(NoteList);            // THE note list of this object
-    return {
-      variant: "editor",
-      notes: list,                                // a document handle is a readable: expose it as state
-      add: ({ text }, ctx) =>
-        self.tx(async (tx) => {
-          const id = uuidv7();
-          (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() });
-          return id;
-        }, ctx),
-      remove: ({ id }, ctx) =>
-        list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx),
-    };
-  });
-});
-

Read host.serve(Notes, factory) as "this host builds Notes objects like this". The factory receives the -normalized input, the caller's principal (chapter III) and self, through which it declares what the object needs. -It returns the members of one variant, tagged with variant.

-

Factories declare; Chord activates. The factory is synchronous and has no side effects. It does not await, open -files or start timers. self.state(NoteList) does not load anything: it declares that the object has this document -and returns a handle that stays valid for the object's whole life. Chord then performs the activation: it loads the -document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for -reload: a reload runs the factory again, and anything the factory did itself would happen twice.

-

Calling it

-
// notes/web.tsx
-export function NotesPanel({ session }: { session: string }) {
-  const notes = useRef(Notes, { session });                 // app-side hook over host.get / release
-  const items = useReadable(notes?.notes)?.items ?? [];
-  return (
-    <section>
-      {items.map((n) => <p key={n.id}>{n.text}</p>)}
-      <AddBox onAdd={(text) => notes?.add({ text })} />
-    </section>
-  );
-}
-

Underneath the hooks is the reference API:

-
const notes = await client.get(Notes, { session: "S1" });   // live and hydrated once awaited
-notes.notes.value.items;                                    // the current list
-notes.notes.subscribe((v) => render(v.items));              // every committed change
-const id = await notes.add({ text: "check the flaky test" });
-notes.release();                                            // the object may end now
-

An awaited reference is live and hydrated: every state member has a value. State members are readables: -.value is the current value, subscribe is called after every commit. The first get for { session: "S1" } runs -the factory; every later get for the same input and caller shares that object. The object ends when the last -reference is released (a type may keep it a little longer with linger).

-
browser tabserverworker S1Notes { S1 }call add { text }relay (header only)strip + validate argsself.tx: push note, commitresult: idstate: ops of the commitresultstate, then result
Figure 3. A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change.
-

Two rules for members

-

Methods return data, never objects. A method that wants to point you to another object returns its address -({ session: "S2" }), and you get it yourself. That keeps every reference owned by someone who releases it.

-

No callbacks as arguments. A callback carries code, and everything it closes over, from one object into another. -Suppose an inbox object passed onFile to another object, which stored it. After a reload of the inbox's plugin, the -other object still calls the old onFile; after the inbox ends, it calls into an object whose file watcher is -already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes -through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII).

-

What is wrong with this version

-

This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the -editor variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as -chapter III shows, a remote object exists per caller, so each caller's Notes object has its own NoteList.

-
Built so far

remote, method, the schema helpers, Static and fingerprints are built (src/next/contracts/). Serving, get, -references and documents are layers L3 to L5.

-
-
- -
III

Who Is Calling

Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members.

-

The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote -what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In -Chord the answer is a principal, a small JSON value the app defines and the links stamp.

-

The app's principal type

-

Chord defines no principal kinds; the app does. Every variant has a kind. Studio uses p2's set:

-
// studio/src/principal.ts
-export type P =
-  | { kind: "owner"; access: "local" | "relay" | "device"; device?: string }
-  | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number }
-  | { kind: "server" }
-  | { kind: "worker"; session: string };
- - - - - - - - - - - - - - - - - - - - - - - -
kindWho
ownerthe person running the app, on loopback, through the relay, or on a paired device
guestsomeone invited into one session, with a mode and the grant (and its revision) that let them in
serverthe server's own code calling a worker
workera session worker calling the server
- -

A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, -and stamps the result on the whole link (server.accept(link, { principal })). The server tells Chord who each worker -is when it resolves it (return { link, principal }). When the server relays a client's request to a worker, Chord -writes the client's principal into the forwarded request; a principal field coming from a client is always -overwritten, and only a trusted parent link may carry one.

-

Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest -who opens it gets a guest record (grant g1) and a cookie; authenticate(req) turns the cookie into -{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 } on every new connection. Chord never sees -cookies or grants, only the finished principal.

-

The caller is part of the identity

-

A remote object exists once per (type, input, principal key). The principal key is a string computed from -the principal; by default its canonical JSON (sorted keys). The app may pass its own principalKey to createHost, -and must pass the same function on every host, or identities split silently.

-
const principalKey = (p: P) =>
-  p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}`
-  : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}`
-  : p.kind === "worker" ? `worker:${p.session}`
-  : p.kind;
-

The design wants the key to include how the client is connected and the grant's revision, so that two access paths -never share objects and a changed grant never reuses the old grant's object:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CallerPrincipal keyNotes { S1 } object
owner, tab A, localowner:local:1
owner, tab B, localowner:local:1 (shared with tab A)
owner through the relayowner:relay:2
guest g1, revision 1guest:g1:r13
guest g1 after a change to write mode, revision 2guest:g1:r24 (new)
-

Variants

-

The contract lists named sets of members; the factory receives the principal and returns the variant that principal -may use:

-
// notes/contract.ts (version 1, final form for now)
-variants: {
-  editor: {
-    state: { notes: obj({ items: arr(Note) }) },
-    methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) },
-  },
-  reader: {
-    state: { notes: obj({ items: arr(PublicNote) }) },     // no `by`: authors are hidden
-  },
-},
-
// notes/worker.ts
-host.serve(Notes, ({ session }, principal, self) => {
-  if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … };
-  return { variant: "reader", notes: … };
-});
-

A reader has no add at all. Chord refuses a call to a member that is not in the caller's variant, so add needs no -check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every -variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per -variant (the reader's notes have no authors). remote(...) enforces them at the type level, and at runtime for -contracts that arrive as data.

-

How a facet knows P

-

P is Studio's type, not Chord's, and every host is created with it (createHost<P>). A plugin's facets are written -and compiled separately, then handed to host.load as values, so the type checker cannot infer P from the host into -the factory's principal. Studio's base code therefore exports its own defineFacet, already bound to P, and every -facet of every plugin imports that one instead of Chord's:

-
// studio/src/facet.ts (base code, exported as @studio/core)
-import { bindFacet } from "@earendil-works/chord/next";
-import type { P } from "./principal.ts";
-
-export const defineFacet = bindFacet<P>();         // a facet's principals are Studio's P
-export type { P, Revocation } from "./principal.ts";
-
// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet)
-import { defineFacet } from "@studio/core";
-
-export default defineFacet((host, self) => {
-  host.serve(Notes, ({ session }, principal, self) => {   // principal: P
-    if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … };
-    …
-  });
-});
-

The import is cheap: @studio/core is a host package plugins already import at runtime (the contract imports Session -from it), P is a type-only import, and bindFacet is a one-line function. The host checks the pairing: load -accepts only facets typed for its own P, so a facet compiled against another app's principal type, or one built with -Chord's untyped defineFacet, does not compile into Studio's start code. Client facets (web.tsx, tui.ts) use the -same defineFacet; they rarely see a principal, because clients serve only local types.

-

None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link -stamped.

-
Review: binding facets to the app's principal type (open)

The design shows defineFacet((host, self) => …) without saying how P reaches the factory. The frozen kernel takes -it as an explicit type parameter, defineFacet<P>(…), defaulting to { kind: string } -(packages/chord/src/next/facet.ts), and the playground writes defineFacet<AppPrincipal>(…). The pre-bound -defineFacet above (bindFacet is a working name) is the owner's preferred direction, recorded as an open point in -packages/chord/docs/next/layer-1.md §4.

-
-

Narrowing on the caller's side

-

Owner and guest run the same web.tsx, and which variant they get is known only at runtime. The types force every -caller to narrow before using variant members:

-
const notes = await client.get(Notes, { session: "S1" });
-notes.add({ text: "hi" });                         // type error: `add` exists only in "editor"
-if (inVariant(notes, "editor")) await notes.add({ text: "hi" });
-expectVariant(notes, "editor");                    // or assert (throws)
-function Editor({ notes }: { notes: RefIn<typeof Notes, "editor"> }) { … }
-

The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the -types gets the same error.

-

Admission before a worker starts

-

Principals also decide whether a caller may reach a host at all. route(Session, { authorize }) runs before Chord -resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for -them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate -this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can -see and do is decided by the variant its factory picks.

-

Inside methods

-

ctx.principal is the caller of the current call. In Studio's notes the editor's add uses it to record the -author:

-
add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx),
-

Where principals act, in summary:

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
PurposeWhere
identity: which object a caller getsprincipal key
variant: which membersthe factory
admission: whether a host is reachableroute.authorize
attribution and fine checksctx.principal in methods
revocationrevokedBy (chapter VII)
-

The bug from chapter II

-

Now the bug is visible. The owner's Notes { S1 } and Ann's Notes { S1 } are two objects (rows 1 and 3 above), and -each declared its own NoteList. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; -they must be views over something that exists once. That is the next chapter.

-
- -
IV

One Store, Many Views

Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it.

-

Every caller has its own Notes object, which is what makes per-caller variants possible. But data must exist once. -The fix is the most common pattern in Chord: a shared store, a local object that holds the data and does the -heavy work once, behind thin per-principal views, the remote objects callers get.

-

local types

-
// notes/contract.ts
-export const NoteStore = local("notes.store", {
-  version: 1,
-  input: obj({ session: str() }),
-  members: {
-    notes: obj({ items: arr(Note) }),                          // state: a schema
-    public: obj({ items: arr(PublicNote) }),
-    add: method(obj({ text: str(), by: str() }), str()),       // methods: method(...)
-    remove: method(obj({ id: str() })),
-  },
-});
-

A local contract has flat members and no variants. The differences:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
remotelocal
reachable fromany host, across linksits own host only
identity(type, input, principal key)(type, input)
principals, variantsyesno
factory(input, principal, self)(input, self)
argumentsstripped to the provider's schema and validated, for every callersee the review note below
values on the same hostpassed by reference, never copiedpassed by reference, never copied
ctx.principalthe callernone
may persist documentsno (chapter V)yes
-

A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote -this note") is passed as data (by), by the view that knows who called. Clients serve only local types, because -nobody can address a client.

-

The pattern

-
// notes/worker.ts
-export default defineFacet((host, self) => {
-  host.serve(NoteStore, ({ session }, self) => {
-    const list = self.state(NoteList);
-    const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) }));
-    return {
-      notes: list,
-      public: pub,                                   // computed once, for every reader
-      add: ({ text, by }, ctx) =>
-        self.tx(async (tx) => {
-          const id = uuidv7();
-          (await tx.draft(list)).items.push({ id, text, by, at: Date.now() });
-          return id;
-        }, ctx),
-      remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx),
-    };
-  });
-
-  host.serve(Notes, ({ session }, principal, self) => {
-    const store = self.get(NoteStore, { session });   // a reference, declared: held for the view's life
-    if (canWrite(principal, session)) return {
-      variant: "editor",
-      notes: store.notes,                             // the store's readable, exposed as-is
-      add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx),
-      remove: (args, ctx) => store.remove(args, ctx),
-    };
-    return { variant: "reader", notes: store.public };
-  });
-});
-
PER-PRINCIPAL VIEWS (REMOTE)SHARED STORE (LOCAL)Notes { S1 }owner · editorNotes { S1 }guest Ann · readerNotes { S1 }guest Bob · readerNoteStore { S1 }notes, public (a derive)self.getself.getself.getAuthority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host.
Figure 4. One store, three views. The redaction runs once in the store; both readers read the same value.
-

self.get(NoteStore, { session }) in a factory declares a reference that the view holds for its whole life; Chord -binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables -directly. Owner, Ann and Bob now give three Notes objects, one NoteStore, one note list and one redacted list.

-

Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. -Base code uses the same pattern with the same primitives; nothing about it is special to core.

-

Derived values, briefly

-

self.derive(name, sources, fn) computes a value from readables and recomputes it when a source changes; viewers get -only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun -that matters: stripAuthor must return the same object for an unchanged note, or every commit costs the size of the -whole list. The tool for that is mapShared.

-

Arguments are checked at the provider

-

Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write -JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to -the provider's own schema (unknown fields removed) and validated with it, whoever the caller is, and never with a -schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves -(instanceof Uint8Array plus an optional length bound), never walked. Stripping copies only when a field was removed; -the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come -from the provider.

-

On the same host nothing is copied: arguments and results are passed by reference. Values are immutable by -convention (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes -values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9).

-
Review: are local arguments validated? (Q15, open)

Design §5's table says local arguments are "neither validated nor copied"; §3 and R7 say arguments are validated -"for every caller". Review round 3 raised this as Q15 (apply R7 to local too, or keep local as the one exception), -and layer-1.md lists it as open. Until it is decided, write local methods so that they do not depend on either -answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view -checking it. chord-next-walkthrough.md concept 6 still describes the older rule ("validated only when the caller is -not a host principal"), which R7 replaced.

-
-

Choosing between the two

-

Use a remote type for what callers address: views, things with per-caller authority. Use a local type for what -exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides -(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a -host-lifetime object, chapter V).

-
- -
V

Writing It Down

Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it.

-

Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker -restart. An object's data lives in documents: named JSON values the object owns, changed only in -transactions. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states -them for Chord.

-

Defining a document

-
// notes/worker.ts
-const NoteList = defineState("notes.list", {
-  schema: obj({ items: arr(Note) }),
-  version: 1,
-  initial: () => ({ items: [] }),
-  persist: true,                                                   // stored; default: memory only
-  checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64,
-});
-
-const Draft = defineState("notes.draft", {                         // one unfinished note per author
-  schema: obj({ text: str() }),
-  version: 1,
-  initial: () => ({ text: "" }),
-  keyed: true,                                                     // many per object, by key
-  persist: true,
-  checkpointWhen: () => true,                                      // small: a full base per write
-});
-

checkpointWhen is required on every definition, with no default (R14, as in Durable since pi 5c1736f82): it says -when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents -checkpoint on every write; growing ones after a number of deltas.

-

A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire -type, not a document type.

-

Singleton and keyed

-
host.serve(NoteStore, ({ session }, self) => {
-  const list = self.state(NoteList);      // THE note list of this object
-  const drafts = self.state(Draft);       // all drafts of this object
-  …
-});
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
singleton (no keyed)keyed
how manyone per objectmany per object, by a short key
loadedat activation, kept in memoryon demand
readlist.value (synchronous)await drafts.get(key): the value or undefined
list(it is one value)for await (const key of drafts.keys()) pages keys
expose as stateyes: the handle is a readableno: expose a method or a singleton index
-

There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large -collection is a method returning pages.

-

Transactions

-

All writes go through self.tx(fn, ctx). Inside, await tx.draft(handle, key?, { tags? }) returns a draft of the -current document, or a draft over initial() if it is absent. Drafts record operations (assignments, push, -Object.assign); asking for the same document twice in one transaction returns the same draft. tx.retire(handle, key?) removes a document. There is no tx.get: reading is drafting, and a draft is an overlay over the current -value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document.

-
saveDraft: ({ author, text }, ctx) =>
-  self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx),
-

Shorthands for one document: handle.change(fn, ctx) (keyed: handle.change(key, fn, ctx)) (loads the document, then runs the synchronous fn on -its draft) and handle.retire(ctx) (keyed: handle.retire(key, ctx)).

-

Four patterns

-

Two decisions (R12 and R13, both as in Durable) shape how transactions are written. Drafts are revoked when the -transaction settles, so a transaction returns at most plain values, never something taken from a draft. And -there is no existence check inside a transaction.

-
// 1. Return data: write in the transaction, read after the commit.
-await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx);
-return list.value.items.length;                    // the committed revision, or a newer one
-
-// 2. Remove if present: retire of an absent document is a no-op.
-await drafts.retire(author, ctx);
-
-// 3. Create with tags if missing: the draft creates it, only if it changes.
-await self.tx(async (tx) => {
-  (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text;
-}, ctx);
-
-// 4. Branch on existence: read first, outside the transaction.
-if ((await drafts.get(author)) === undefined) { … }
-

Durable's rule

-

Inside the callback, only await tx.*. No other objects, no network, no models, no processes, no second self.tx, no -reads past tx. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch -first; decide and write inside.

-
// Wrong: a fetch inside the transaction holds the object's line.
-await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx);
-// Right: fetch, then write.
-const title = await fetchTitle(url, ctx.signal);
-await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx);
-

What a commit does

-
INSIDE SELF.TXAFTER THE WRITEwait for the lineper objectrun the callbackdrafts record opsprepareDelta checks placed valuesstorage batchdeltas or a baseadoptpointer swapnotify locallyonce, with the opssendone batch per linkrevoke draftstouching one throwsstoredAn error or an abort before the storage write discards everything. Storage commits are serialized per host.
Figure 5. The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit.
-

Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same -host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the -document again; chapter XVII shows how one commit touching several holds still arrives as one message.

-

Tags and deleting by tag

-

Tags are typed tokens set when a document is created:

-
export const SessionTag = defineTag<string>("studio.session");     // base code, shared by every plugin
-await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx);
-// when the session is deleted
-await store.retireByTag(SessionTag, "S1");                          // every document tagged S1
-

The design names the operation store.retireByTag(tag, value) without saying who holds the store; session delete is -one of the app decisions still open (#18, chapter XIX).

-

Who may persist

-

persist: true is allowed only in local objects and lifetime: "host" objects: identities without a principal. A -remote object's identity contains the principal key, which changes with the access path and the grant revision. Its -stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type -level (self.state in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a -stable identity: put the documents into a local shared store"). This is the second reason for the shared store.

-

What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as -their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else -(natives, derives, routines, references) is rebuilt on activation.

-

Facet documents

-

A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as -preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the -object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served -object's documents.

-

Ownership across plugins

-

No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and -methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is -created (as with Durable's createConversation({ init })), never written into core's documents.

-

Versions and migration

-

A change of shape or meaning raises version and adds a pure migrate(old, from):

-
const NoteList = defineState("notes.list", {
-  schema: obj({ items: arr(NoteV2) }),           // NoteV2 adds `pinned`
-  version: 2,
-  migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }),
-  initial: () => ({ items: [] }),
-  persist: true,
-  checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64,
-});
-

Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed -ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only -transaction writes it.

-

Chord does not walk values against schemas. Instead, each definition computes a schema fingerprint once -(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord -compares them: the same version with a different fingerprint fails the object broken ("schema changed without a -version bump"). So every structural change needs a bump, an additive one with a trivial migrate. The one exception -to "no value checks" (R8): the result of migrate and of initial() is validated against the new schema once, when -produced. A failed check, a throwing migrate, or old code meeting a newer document refuses the access, keeps the -stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on -you.

-

Storage

-

Behind the store is a DocumentStorage, passed as createHost({ storage }); memory by default. The server has one -shared storage, each worker one for its session.

-
interface DocumentStorage {
-  commit(writes: readonly DocumentWrite[]): Promise<Seq>;            // create/change/retire, atomic
-  find(address: DocumentAddress): Promise<DocumentRecord | undefined>;
-  load(id: string): Promise<StoredDocument | undefined>;             // version, value, deltasSinceBase
-  scan(query: { owner: string; definition: string }, limit: number, cursor?: string):
-    Promise<{ keys: string[]; next?: string }>;
-  byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>;
-  mintId(): Promise<string>;
-}
-

Chord ships a memory storage and a SQLite storage (over @earendil-works/chord/sqlite, the driver layer that moved -from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address -occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. -Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over -Durable's. Your own storage can prove itself with registerDocumentStorageConformance(runner, name, withStorage) from -@earendil-works/chord/testing.

-

Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next -access. A storage error ends the store and closes the host; onError receives { kind: "storage_failed" } and the -app decides (chapter I).

-

Kinds of state

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
KindForAPI
documentdata the object ownsself.state, self.tx
derivecomputed valuesself.derive (chapter VIII)
source readableexternal data: Durable views, files, logsvalue plus ordered changes; feeds derives
paged methodlarge or queryable collectionsa method returning pages (chat history)
plugin databasefull-text search, large blobsa native opened by its owner (chapter IX)
-

App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives -in Durable.

-
Review: does an absent singleton read undefined or initial()? (Q11, open)

Design §9 says "whether a document exists is read outside the transaction (handle.value / handle.get give -undefined for an absent one)" and also "a singleton that does not exist yet reads initial() through its handle; -retiring resets it to initial()". Review round 3 proposes: singletons always read initial(), only absent keyed -documents read undefined, and singleton existence needs no API. This guide follows the proposal (list.value is -never undefined); it is not decided.

-
-
Built so far

Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel -has a working version with a JSONL storage, which the design rejected.

-
-
- -
VI

Holding On

References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code.

-

A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and -started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the -whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a -reference does; plugin code contains no retry loops and no request ids.

-

Three ways to hold

- - - - - - - - - - - - - - - - - - - - - - - -
WhereAPIReleased
a factoryself.get(Type, input)when the object ends
a callawait ctx.get(Type, input, { state? })when the call ends
outside objectsawait host.get(Type, input, { signal, state?, parkTimeout? })ref.release()
-

Use self.get for what an object needs its whole life (the view's store), ctx.get for what one call needs, and -host.get only in UI code, tests and entry points. { state: false } binds for methods only, without receiving state. -A reference keeps its object alive while held.

-

The object lifecycle

-
(get) → activating → live → ending → ended
-             └→ failed (backoff) → activating …
-

Activating: Chord binds the declared references and waits until they are live and hydrated, loads singleton -documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object -goes live when all of that succeeded. Calls wait behind its gate until then.

-

Ending: routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, -references are released. An object with lifetime: "referenced" (the default) ends when its last reference goes, -after the type's linger if it has one. A lifetime: "host" object (only for types without a required input) lives -as long as the host. A running, finishing or blocked job counts as a reference (chapter XI).

-

ref.state

-

Every reference reports what is going on as a tagged value; listeners get each new one:

-
type RefState =
-  | { kind: "loading" }                                                // first hold, reconnecting, restarting
-  | { kind: "live" }
-  | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry
-  | { kind: "released" };
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SituationCallsValues
link droppark until reconnectedreadable, loading
object loading or restartingpark until livereadable, loading
rollout: provider ahead or installingpark until installedreadable
object failedreject at once with the causefailed, with cause and retryAt
-

Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 -s by default, parkTimeout per get), then fail unavailable. Otherwise a call ends only through its caller's -signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is -not resent: Chord cannot know whether it took effect.

-

Errors look like disconnects. A consumer handles a failed call exactly as a dropped link: show the error, let the -user retry. Work that must not run twice carries its own idempotency key as data (Durable's requestId, a job's -key).

-
The worker restarts while a tab watches the notes

Watching

The tab holds Notes { S1 }. ref.state is live; notes.notes.value has 40 notes.

-

The worker goes away

The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its -hold is gone; the tab's reference goes loading. The 40 notes stay readable.

-

A click meanwhile

Ann's colleague clicks Add. The call parks on the reference; nothing is sent yet. The deadline is 60 s.

-

Chord resolves again

Holds on S1 still exist, so Chord calls route(Session).resolve({ id: "S1" }) again, with the per-address backoff. -The app forks a new worker: a new incarnation of Session({ id: "S1" }).

-

Re-hold

The server re-holds every hold on S1 over the new link. The new worker activates NoteStore { S1 } (its notes load -from storage) and the views; it sends fresh snapshots. The tab's reference goes live.

-

The parked call runs

The parked add is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code.

-
-

Failure and retry

-

Any broken part fails the whole object: a half-working object could break rules that rely on its documents -changing together. Chord tags the cause by where it failed:

-
type FailureCause =
-  | { kind: "transient"; error: ChordError }   // an open threw, a dependency was unreachable: automatic backoff
-  | { kind: "broken"; error: ChordError };     // factory, derive, hydration or migrate threw: retried only by a trigger
-

Transient causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success -resets it. Broken causes would fail the same way again (a code bug, invalid data), so they are retried only by a -trigger: a plugin reload, ref.retry() (a Retry button), or a readable the factory declared with -self.retryOn(...readables). Only an opener may declare its own failure broken, with throw broken(error), for -example for invalid configuration. A failed reference stays held; on success it goes loading, then live.

-
host.serve(EnvConnection, ({ env }, self) => {
-  const config = self.get(EnvConfig, {});                 // a facet or registry value, outlives the attempt
-  self.retryOn(config.envs);                              // a config change retries a broken open
-  const ssh = self.open(async ({ signal }) => {
-    const c = config.envs.value[env];
-    if (!c) throw broken(new Error(`unknown environment ${env}`));
-    return connect(c, signal);
-  }, (s) => s.close());
-  …
-});
-

retryOn sources must live independently of the attempt (facet documents, registry values); a member of one of the -object's own references is a type error, since it goes away with the attempt.

-

Waiting for dependencies

-

An object whose references are not live yet stays loading and goes live the moment they are, bounded by -readyTimeout (60 s). The decision for the failure case (R18) is what makes recovery automatic:

-
tabNotes { S1 }NoteStore { S1 }holdself.get (activation)open fails: disk not mountedfailed (transient)failed: transient, cause insidethe view keeps its reference; its state is the retry triggerbackoff retry: open succeedsliveretried at once (backoff cancelled)live
Figure 6. Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live.
-

A dependency that fails, or does not become live in time, fails the dependent transient with the dependency's -cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord -uses their state as a retry trigger. Factory authors write self.get and know none of this. After an object is live, -a dependency leaving live does not change it: its calls that use the dependency get the dependency's error, or park -while it loads.

-

Cycles are rejected (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. -Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and -across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation -broken, naming it ("A → B → A"), and keeps no references.

-

No optional references

-

get has no optional flag. When a dependency's plugin may be missing, turn the dependency around: the dependent -owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is -optional, so radius owns radius.forwards and ports contributes its forwards; without ports the table is empty and -radius works (chapter XIII).

-

ctx

-

Every call gets a ctx as its last argument:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldMeaning
signalfires when the caller cancelled or the object ended; pass it to everything you wait on
principalthe caller (remote types only)
incarnationthe calling host's incarnation, for calls that arrived over a link (chapter X)
getreferences released when the call ends
telemetry, trace(name, attributes, fn)the current span; a child span with a child ctx (chapter XVII)
jobId, progressin a job's run (chapter XI)
-
exportText: async ({}, ctx) => {
-  const catalog = await ctx.get(Catalog, {});                     // released when the call ends
-  const title = catalog.sessions.value[session]?.title ?? session;
-  const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal });
-  return await res.text();
-},
-

There are no code versions and no request ids in ctx. Durable's Context is the same module (chord/context): -ctx extends it, so passing ctx into pi-durable needs no conversion. Code outside objects gets a ctx from -host.root(fn, { principal?, signal? }); inside Durable tools it uses chord(fn, context) (chapter XIV).

-
Built so far

References, ctx and the lifecycle are layers L4 and L5. ChordError and FailureCause are specified in the L1 -draft (layer-1.md), with codes invalid, contract_mismatch, forbidden, gone, not_ready, released, -too_large, unavailable and aborted, and a finer reason such as invalid/args.

-
-
- -
VII

Taking Access Away

Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back.

-

The owner removes Ann from S1. Her tab still has a WebSocket open and a Notes { S1 } view on worker S1. Both must end -now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the -mechanism; the app decides who is blocked and remembers it.

-

Labels, not predicates

-

The app declares, once, which revocation labels each principal carries. Revoking means revoking one label; -everything carrying it is hit.

-
// studio/src/principal.ts
-export type Revocation = { grant: string } | { device: string } | { session: string };
-
-export const revokedBy = (p: P): Revocation[] =>
-  p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }]
-  : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }]
-  : [];
-
-const server = createHost<P, Revocation>({ address: Server(), principal: { kind: "server" }, revokedBy });
-// every worker is created with the same revokedBy
- - - - - - - - - - - - - - - - - - - - - - - -
PrincipalLabels
guest Ann (grant g1) in S1{ grant: "g1" }, { session: "S1" }
guest Bob (grant g2) in S1{ grant: "g2" }, { session: "S1" }
owner on the paired phone{ device: "phone" }
owner, localnone
- - - - - - - - - - - - - - - - - - - -
CallHits
server.revoke({ grant: "g1" }): remove AnnAnn
server.revoke({ session: "S1" }): archive S1Ann and Bob
server.revoke({ device: "phone" }): sign out the phonethe phone
-

A label is plain data because revocation is an event that must travel from the server to the workers; a predicate -function could not be sent.

-

What revoke does

-

host.revoke(r) ends every object and closes every link whose principal p has r in revokedBy(p) (compared by -canonical JSON; never the host's own principal), and passes r to its children, which apply their own revokedBy.

-
owner tabserverworker S1Ann's tabguest g1Sharing.remove({ grant: "g1" })app: mark g1 blocked (document)revoke({ grant: "g1" })close linkrevoke { grant: "g1" }end Notes { S1 } as g1reconnectauthenticate: g1 blockedrefused (4401)
Figure 7. Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record.
-

On worker S1, for revoke({ grant: "g1" }):

- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ObjectPrincipalHit
Notes { S1 }owner, localno
Notes { S1 }guest g1yes: ends
Notes { S1 }guest g2no
NoteStore { S1 }none (local)no
-

Ending Ann's view releases its reference to the store, which keeps serving the others.

-

Chord persists nothing here

-

revoke cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which -authenticate checks on every new connection. After a server restart nothing live exists to revoke; every client -authenticates again and blocked grants are refused there.

-

No re-stamping. A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, -her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references -re-hold by themselves, now as an editor. Reconnecting never restores revoked authority.

-

Revocation beats jobs (R11). Normally a running job keeps its object alive after the last reference is released -(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the -abort signal and the normal close timeout.

-

Work in shared stores

-

Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs on -behalf of someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by revoke. -Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine -(chapter X):

-
// in a shared store that runs work for guests
-const sharing = self.get(Grants, {});                              // the sharing plugin's live grants
-self.routines.on(sharing.live, async (live, ctx) => {
-  for (const run of shell.running.value)
-    if (run.args.grant && !live[run.args.grant]) shell.abort(run.id);   // idempotent
-});
-

There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state -cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs.

-

Who does what

- - - - - - - - - - - - - - - - - - - -
ChordThe app
stamps principals on linkswhat a grant is; share links, cookies, authenticate
keeps one object per principal keydefines Revocation and revokedBy
on revoke(r): ends matching objects and links, passes r downdecides when to revoke; stores who is blocked
-
- -
VIII

Computed Values

Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking.

-

The reader's notes without authors, a note count, the number of people viewing: values computed from other values. -Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A -derive is computed once per change and sent to viewers as a difference.

-

self.derive

-
const count = self.derive("count", [list], ([l]) => l.items.length);
-const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({
-  total: l.items.length,
-  pinned: pinned.length,
-}));
-

self.derive(name, sources, (values, previous) => value). Sources can be documents, readables, other derives and -state members of references. The value is recomputed when a source changes and diffed into operations for the wire. -Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer -never sees a note list and a count from different commits.

-

previous is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a -host teardown, so a derive must be correct without it. A derive that throws fails its whole owner broken.

-

How the difference is found

-

Chord compares the new value with the previous one using Delta's diffRevisions. Parts that are the same objects -as last time cost one === each. Parts that are new objects must be compared by content.

-

The footgun. A derive that builds new objects for unchanged data makes every commit cost the size of the whole -value, per view. The reader list from chapter IV, written naively:

-
self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) }));
-// one note added to 2,000: 2,001 new objects, every one compared by content
-

mapShared(f) keeps one result per source element: f runs only for new or changed elements, unchanged ones -return their previous result object.

-
import { mapShared } from "@earendil-works/chord/delta";
-
-const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at }));
-self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) }));
-// one note added: one new object, 2,000 identical references; the diff emits one insert
-
Review: mapShared is planned, not available

Design §12 introduces mapShared as coming "from @earendil-works/chord/delta", but nothing in packages/chord/src -defines it yet, and Delta does not yet treat bytes as atomic leaves in diffRevisions. Design §28 lists both as the -step "Delta in pi", before the layered rebuild. Until then, keep a Map from source element to result yourself, which -is what mapShared will do.

-
-

Cookbook

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CaseHow
select a field, look up by key, a small aggregatecompute it; the value is small
filter or map a listmapShared over the elements
redactionpreferably separate documents per visibility; otherwise mapShared, computed once in the shared store, never per view
join two sourcesmapShared plus the looked-up source's revision
a window over a large listtake a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends
a large view combined from several sourcesre-path the sources' operations, as Durable's ConversationView does; application code never interprets ops
-

Container width (R24): structural sharing is O(change) along a path, not across a container's width. Changing one -item of a flat array copies the array's slots, and mapShared visits every element once (one identity comparison -each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented -limit.

-

Readables that are not documents

-

Three small readables round out the toolkit:

-
    -
  • constant(value): never changes. Start data uses it (chapter IX).
    • -
  • self.writable(initial): a host-local readable of any in-process value (functions included), compared by identity, -with set. It never goes on the wire; it exists for registry contributions that change (chapter XIII).
    • -
  • Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and -Chord v2 share the readable type (chord/src/state.ts), so it can feed a derive or be exposed as state directly.
    • -
-
Review: exposing a writable (Q14, open)

A writable may hold functions; nothing yet stops a factory from returning one as a state member, which would reach -the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked -when the object is served. Undecided.

-
-

Presence: who is looking

-

host.references(Type) is a readable of who holds references to objects of a type on this host: input, principal and -client. Client holds carry an opaque origin id, so two tabs are two clients.

-
host.serve(NoteStore, ({ session }, self) => {
-  const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) =>
-    refs.filter((r) => r.input.session === session).length);
-  return { …, viewers };
-});
-

A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the -UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There -is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's linger keeps a -just-closed object warm, and that also keeps the worker alive through host.activity.

-

"Leaving" is never derived from an unmount. A UI reload unmounts and remounts components; a component unmounted by -a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, -viewer and login-abandon leases with one mechanism.

-
Built so far

Readables and derives are layer L2 (a readable is today's replicatedState, derives reuse today's derived cells over -it). Not started in the rebuild.

-
-
- -
IX

Things the OS Gives You

Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads.

-

Studio's notes get an inbox: a text file dropped into the session's inbox/ directory becomes a note. That needs a -file watcher, a native: an OS-level value (a file handle, a socket, a process, a database connection) that cannot -be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data -directory, which the plugin does not know. This chapter covers both.

-

Start data is an object

-

A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start -message. There is no special mechanism for them. Start data is an ordinary local object with lifetime: "host" and -constant state, served by base code's core facet:

-
// studio/src/core/contract.ts
-export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) });
-export const SessionInfo = local("studio.session-info", {
-  version: 1,
-  input: obj({}),
-  members: { init: Init },
-});
-
-// studio/src/core/worker.ts
-export const coreWorkerFacet = (init: Static<typeof Init>) => defineFacet((host) => {
-  host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" });
-  // … the Harness and other core objects (chapter XIV)
-});
-

Any plugin on the worker declares a reference and reads it after activation: in an opener, a derive, a routine or -a method, never synchronously in the factory.

-
const info = self.get(SessionInfo, {});
-// later: info.init.value.id, info.init.value.dataDir
-

Paths are derived by the plugin from generic start data (join(dataDir, id, "notes-inbox")), never provided per -plugin. Platform bindings work the same way: on a Durable Object, a core object's self.open returns the binding -(state, env) and its close does nothing.

-

self.open

-
const watcher = self.open(
-  async ({ signal }) => watchDir(path, { signal }),       // open: during activation
-  (w, reason) => w.close(),                               // close: when the object ends or restarts
-  { closeTimeout: 10_000 },                               // optional; host default 30 s
-);
-watcher.value;                                            // the watcher; reading it before activation throws not_ready
-

self.open returns a handle with .value (type Native<T>), not a proxy. The rules:

-
    -
  • Natives live only in objects. Not at module level, not in the facet. A module-level watcher would survive a -reload with nobody to close it.
    • -
  • Order: natives open one after another in declaration order (a later opener may read an earlier handle's -.value) and close in reverse. The object goes live after all opens succeeded.
    • -
  • The value never changes during an attempt. No replacement, no versions, no stale-handle logic. To change a live -native, the object restarts.
    • -
  • A failed open makes the object failed: transient with backoff by default, broken if the opener says so -(throw broken(error)). A partly failed activation closes the natives already open, with reason end; the next -attempt opens all of them anew.
    • -
  • close gets a reason: { kind: "reload" } (the plugin reloads and this native object restarts), { kind: "stop" } (the host shuts down) or { kind: "end" } (the last reference ended, or the type was removed). Each -close is bounded by closeTimeout, then reported as a possible leak while the rest continues. A throwing close -does not stop the others; host.close() collects the errors, background ends report to onError.
    • -
  • Behaviour around a native goes into routines (chapter X). Listeners on natives outside routines are forbidden.
    • -
-

An object with at least one self.open is a native object; one without is a pure object. The difference -decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart.

-

The inbox

-

The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a lifetime: "host" -object. The facet cannot self.get a type it serves, and the inbox has no meaningful input, so it is an object -without input that reads its session from the start data:

-
// notes/contract.ts
-export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} });
-
-// notes/worker.ts
-host.serve(NoteInbox, (_input, self) => {
-  const info = self.get(SessionInfo, {});
-  const watcher = self.open(async ({ signal }) => {
-    const { id, dataDir } = info.init.value;                       // read in the opener: after activation
-    return watchDir(join(dataDir, id, "notes-inbox"), { signal });
-  }, (w) => w.close());
-
-  self.routines.loop("import", async (ctx) => {                    // chapter X
-    const store = await ctx.get(NoteStore, { session: info.init.value.id });
-    for await (const file of watcher.value.files(ctx.signal)) {
-      await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx);
-      await rm(file);
-    }
-  });
-  return {};
-}, { lifetime: "host" });
-

Note ctx.get(NoteStore, …) in the routine rather than self.get in the factory. The factory runs before the -session id is known (start data is read after activation), so a host-lifetime object without input gets -session-scoped objects with ctx.get in a routine or method (R22). A get whose input is computed after activation is -to be added only if a real case needs it.

-

Natives with work in flight

-

A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, -use one of two patterns:

-
    -
  • Drain in the background. close detaches the old resource and returns at once; it disposes of it when its work -finishes, or after closeTimeout at most. Chord opens the new one immediately. Code mode's sandbox does this: a -script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn -fails), new scripts start on the new one.
    • -
  • Swap through a registry. The resource is a registry contribution (chapter XIII). Readers always look up the -current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline.
    • -
-

Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a -sandbox) is the owner's business: close may drain instead of tearing down. What an old native does after its owner -restarts is also the owner's close: a model runtime keeps working until collected, a closed SSH connection fails its -in-flight exec.

-

Natives that follow a relocation

-

A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be -heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its -internal resource and bumps a generation, so late events from the old location are dropped:

-
const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close());
-self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox")));
-

Call-scoped natives

-

A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a -download) is opened and closed inside the method with try/finally. There is no ctx.open:

-
list: async ({ path }, ctx) => {
-  const session = await ssh.open(config, ctx.signal);
-  try { return await session.readDir(path, ctx.signal); }
-  finally { session.close(); }
-},
-

Core vs. plugin

-

SessionInfo comes from base code, NoteInbox from a plugin; both use the same primitives. Where code lives decides -which it is. Base code (the app's src/, imported statically by the start code under a fixed name such as -@studio/core) is always there, not switchable, and a change restarts the hosts. A plugin (its own directory, -found by listing, named by its package.json) is switchable and reloads. Core is what the app always needs and -everything builds on: the harness, the session list, access, the session location. Optional features are plugins; -code mode is a plugin.

-
Built so far

Natives and host-lifetime objects are layer L4. constant is decided (R23), not built.

-
-
- -
X

Standing Behaviour

Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction.

-

Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived -session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare -setIntervals and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord -has one declared form for it: the routine.

-

Four kinds

-
self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx));   // due time survives reloads
-self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx));        // a readable or event source
-self.routines.loop("import", async (ctx) => {                                     // runs until the object ends
-  while (!ctx.signal.aborted) await importNext(ctx);
-});
-self.routines.custom("ticker", ({ signal, run }) => {                             // anything else
-  const timer = setInterval(() => run(tick), 1000);
-  return () => clearInterval(timer);                                              // cleanup undoes the setup
-});
-

Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their -plugin. Each run gets its own ctx; errors are caught and reported, never uncaught. Routines are named (on -optionally by its source) so Chord recognizes them across reloads; that is how every keeps its due time.

-

Serialized per routine

-

The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at -most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported.

-

The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through -restart(W1) when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and -restarts it a second time.

-

Reconcile, don't track

-

A routine reconciles from the current value: it makes the world match what it sees now. It writes no "in -progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value -compares with the current value itself; there is no overlap option. Subscribing to the object's own documents is -gapless: every commit is seen.

-

No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an -object's behaviour when it ends or reloads.

-

Rules between two owners

-

Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived -session are read-only" involves the catalog (which knows archiving) and NoteStore (which holds notes). Two tempting -designs are wrong:

-
    -
  • A cross-owner transaction (archive and lock the notes atomically) would make the catalog know every plugin and -couple their storage.
    • -
  • A notification (catalog calls notes.archive(session)) can be lost: the worker is not running, the link drops, -the process crashes between the archive and the call.
    • -
-

The design's rule: the owner of the dependent data enforces the rule by reconciling from the other owner's state -with an idempotent routine.

-
// NoteStore: an archived session's notes are read-only
-const flags = self.state(NoteFlags);                         // { archived: boolean }, NoteStore's own document
-const catalog = self.get(Catalog, {});                       // the server's session list: another owner
-self.routines.on(catalog.sessions, async (sessions, ctx) => {
-  const archived = sessions[session]?.archived ?? false;
-  if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx);
-});
-// add and remove refuse while flags.value.archived
-

After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The -same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked -(chapter VII), and keeps any invariant between plugins.

-

Reading without starting a worker

-

The session list in the sidebar shows each session's note count. Getting NoteStore { S1 } for every session would -start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker -side reports a summary to a server-side object by a method call from a routine, and clients read the server object.

-
// notes/contract.ts
-export const NoteCounts = remote("notes.counts", {
-  version: 1, input: obj({}), placement: () => Server(),
-  variants: {
-    owner: { state: { counts: map(int()) } },
-    worker: { methods: { report: method(obj({ session: str(), count: int() })) } },
-  },
-});
-
-// notes/server.ts
-const Counts = defineState("notes.counts", {
-  schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }),
-  version: 1, initial: () => ({ bySession: {} }), persist: true,     // survives a server restart
-  checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64,
-});
-
-host.serve(NoteCountStore, (_input, self) => {                        // local, lifetime "host"
-  const counts = self.state(Counts);
-  return {
-    counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)),
-    report: ({ session, count, incarnation }, ctx) =>
-      counts.change((d) => {
-        const last = d.bySession[session];
-        if (last && last.incarnation > incarnation) return;               // uuidv7: an older worker, ignore
-        d.bySession[session] = { count, incarnation };
-      }, ctx),
-  };
-}, { lifetime: "host" });
-
-host.serve(NoteCounts, (_input, principal, self) => {
-  const store = self.get(NoteCountStore, {});
-  if (principal.kind === "worker") return {
-    variant: "worker",
-    report: ({ session, count }, ctx) => {
-      if (session !== principal.session) throw forbidden();               // a worker reports only itself
-      return store.report({ session, count, incarnation: ctx.incarnation }, ctx);
-    },
-  };
-  return { variant: "owner", counts: store.counts };
-});
-
-// notes/worker.ts, inside NoteStore
-const counts = self.get(NoteCounts, {});                                   // a reference to the server
-self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx));
-

The server object keeps the latest value per session (in a document if it must survive a server restart) and -ignores reports from an older worker incarnation (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the -fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): ctx.incarnation, -next to ctx.principal, not part of the principal or its key, never sent as data, so it cannot be forged. Session -status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no -caller incarnation of its own, so the view passes it on as data.

-
Review: ctx.incarnation across a relay (Q13, open)

A worker's call to the server arrives directly, so ctx.incarnation is the worker's. But a call relayed through the -server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's -handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp -the principal. Undecided; the pattern above only needs the direct case.

-
-
Built so far

Routines are layer L8. The design treats every as one of four forms over one runner; review round 3 proposed cutting -to on and loop only (C4), not decided.

-
-
- -
XI

Work That Outlives the Call

Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome.

-

Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be -cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do -this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work -that must resume after a crash. Chord's form for it is the job, built in pi-durable's sandwich: intent (a -transaction), effect (anything), outcome (a transaction in the current code).

-

Defining and running a job

-
// notes/worker.ts
-const ImportJob = defineJob("notes.import", {
-  version: 1,
-  args: obj({ path: str() }),
-  progress: obj({ lines: int() }),
-  result: obj({ added: int() }),
-  concurrency: 1,                                         // one import at a time per NoteStore
-});
-
-const ImportResults = defineState("notes.import-result", {
-  schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }),
-  version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true,
-  checkpointWhen: () => true,
-});
-
-host.serve(NoteStore, ({ session }, self) => {
-  const list = self.state(NoteList);
-  const results = self.state(ImportResults);
-  const imports = self.job(ImportJob, {
-    run: async ({ path }, ctx) => {                                       // 2. effect
-      let lines = 0;
-      for await (const line of readLines(path, ctx.signal)) {
-        await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx);   // keyed by job id
-        ctx.progress({ lines: ++lines });
-      }
-      return { added: lines };
-    },
-    finish: async (outcome, tx) => {                                      // 3. outcome, current code
-      (await tx.draft(results, outcome.id)).outcome =
-        outcome.kind === "done" ? outcome.result : { error: outcome.error };
-    },
-  });
-  return {
-    …,
-    imports: imports.running,                                             // live list of runs
-    startImport: ({ path, requestId }, ctx) =>
-      imports.start({ path }, ctx, { key: requestId }),                   // 1. intent (refined below)
-    abortImport: ({ id }) => imports.abort(id),
-  };
-});
-

defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? }); progress and the outcome -are typed by the schemas.

-
tabNoteStore { S1 }runeffectstartImport({ path, requestId })1. intent: busy check, dedupe by key, commit{ id }start after the commitread lines, add notes (keyed by jobId)progress { lines } (throttled)the tab may close here: the job counts as a referencesettles: { kind: "done", result }3. record outcome, finish(outcome, tx), ack
Figure 8. The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code.
-

The three steps

-

1. Intent. job.start(args, ctx, { key? }) is shorthand for tx.start(job, args, { key? }) inside a -transaction, which may contain other writes. The busy check (concurrency) and deduplication (the same key returns the -existing run) are atomic and in the same commit; run starts only after the commit. Run ids are uuidv7.

-

2. Effect. run(args, ctx) may do anything: processes, network, other objects. It writes no document itself; it -returns the outcome. External writes carry a dedupe key derived from ctx.jobId (above: note ids), so an interrupted -attempt cannot apply them twice. On a reload run keeps its old code.

-

3. Outcome. When run settles, Chord records the outcome in its own record, then calls the current code's -finish(outcome, tx) and acknowledges the run in the same commit. The outcome is tagged:

-
type Outcome<R> = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" });
-

A throwing finish changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. finish -is optional and follows Durable's transaction rule.

-

Dedupe ends with the run

-

Deduplication covers unsettled runs only (Q9). Once finish has committed, Chord drops the run's record and key -and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the -owner's. Studio records which run a request started, in the same commit as the intent, and finish records the -result by run id (ImportResults above):

-
const Requests = defineState("notes.import-request", {
-  schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true,
-  checkpointWhen: () => true,
-});
-
-startImport: async ({ path, requestId }, ctx) => {
-  const prior = await requests.get(requestId);                  // read outside the transaction (R13)
-  if (prior) return { id: prior.run };                          // started before, maybe finished long ago
-  return self.tx(async (tx) => {
-    const run = await tx.start(imports, { path }, { key: requestId });   // dedupes a concurrent retry
-    (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id;
-    return { id: run.id };
-  }, ctx);
-},
-

Two concurrent retries both miss prior, but the key makes tx.start return the same unsettled run. The client -watches imports for progress and reads the result by run id once it is gone from running.

-

Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a -retention limit would let a retried key rerun its effect. The owner knows which results matter.

-

Watching, waiting, aborting

-
    -
  • running is a readable of { id, version, args, progress, phase }. phase is running, finishing (outcome -recorded, finish not yet committed) or blocked (with the reason). concurrency counts only active runs. The -panel shows a progress bar from it.
    • -
  • Progress is in memory, throttled and sent as deltas. For output, use OutputBuffer and Progress from -@earendil-works/chord/output: bounded windows, adaptive throttling, no write amplification.
    • -
  • Waiting: start returns a run with outcome(ctx). The caller's abort ends only its wait, never the run.
    • -
  • Abort is a request: job.abort(id) fires the run's ctx.signal; the run decides what it returns (a result -that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. finish runs in -Chord's own context.
    • -
-

Lifetime, persistence, revocation

-

Jobs keep their owner alive. A running, finishing or blocked job counts as a reference to its object, which ends -only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also -count as activity for the worker's keep-alive.

-

Revocation takes precedence (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs -get the abort signal and the normal close timeout. A job runs as its object's principal.

-

Persistence: jobs are in memory by default and gone with the process. persist: true (only for stable -identities) keeps the record: a run live at a crash ends lost at the next activation and finish records it. -Nothing is rerun automatically.

-

Versions: a run started under an older job version keeps its version; finish and running see it through pure -accepts adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the -reload: the old outcome is blocked, kept and reported until a later reload brings the adapter.

-

Native objects: a job in a native object ends aborted when the object restarts on a reload (chapter XV). In a -pure object it keeps running on its old run code.

-

What is not a job

- - - - - - - - - - - - - - - - - - - - - - - -
WorkUse
request and response that may die with its callera method
input into running work (a pasted code, an approval)a flow (chapter XII)
work that must resume after a crasha Durable task
standing behavioura routine
-
Review: jobs are the largest open simplification (#9, C3)

Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to -address and watch (LoginFlow { id }). Options: (a) jobs stay records inside their owner, watched through running; -(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: -shrink jobs to a runner over documents (no job versions, accepts, blocked, persist) or move them out of the -kernel. Not decided. chord-next-walkthrough.md concept 17 still shows the older outcome shape { ok: true, result }; -the design tags outcomes by kind.

-
-
- -
XII

Asking a Person

Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice.

-

A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's -approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any -tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a flow kept as its -owner's state, and the Questions service, a shared store for questions.

-

A flow is state, not a call

-

The tempting design is a long-held server call, step(), that blocks until the next step. A link drop fails it, and -the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as state, which -re-holds by itself after any drop:

-
// models/contract.ts (sketch from the design)
-start:  method(obj({ provider: str() }), obj({ id: str() }))              // uuidv7
-answer: method(obj({ id: str(), step: int(), value: str() }), Answered)   // ok | "step no longer active"
-cancel: method(obj({ id: str() }))
-
-// LoginFlow { id }: state
-type FlowState =
-  | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] }
-  | { kind: "done" | "failed" | "cancelled"; notices: Notice[] };
-

The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A -promise-based callback inside the work (pi-ai's prompt during a login) stays inside the owner: its job keeps the -resolver, and nothing crosses an object boundary.

-

Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a -second protocol:

-
async function* steps(flow: LoginFlowRef, ctx: Ctx) {
-  let last = 0;
-  for (;;) {
-    const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx);
-    if (s.kind !== "running") return s.kind;
-    last = s.step!.id;
-    yield s.step!;
-  }
-}
-
-for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx))
-  await login.answer({ id, step: step.id, value: answerFor(step) }, ctx);
-

last keeps steps from being lost or repeated across re-holds.

-
Review: where a flow lives (#11, deferred)

Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object LoginFlow { id } -living until done; tied to #9 (jobs as objects). The LoginFlow { id } address above assumes (b). Also undecided is -how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped -reference, never an unmount (chapter VIII).

-
-

The Questions service

-

The Questions service (apps/p2/docs/questions.md) is a p2 design, not a Chord feature, built from Chord's -primitives: a shared store with per-principal views. The asker gets ask; viewers get the open questions and -answer.

-
export type Question =
-  | { kind: "text"; message: string; secret?: boolean; help?: string }
-  | { kind: "select"; message: string; options: readonly { id: string; label: string }[] }
-  | { kind: "confirm"; message: string; allow: string; deny: string };
-
-type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" };
-
-ask(question, {
-  audience,                // a readable or a list of principals
-  policy,
-  key?,                    // stable per asking flow: a re-ask reattaches to the pending question
-  deadline?,               // then resolve with what is there; `missing` names who did not answer
-}, ctx): Promise<Result>;
-

Its rules:

-
    -
  • The store is the truth. Pending questions and their resolvers live in the store; open is a readable over it, -filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question.
    • -
  • One settlement point. Register the waiter and the abort listener before publishing the question; an answer, an -abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer -active". Answers are validated against the stored question (a select id must be one of the options). Never await an -answer inside a transaction.
    • -
  • Policies over a live audience. first resolves with the first answer; quorum(n) after n answers; all when -every member of the audience has answered. With join: "live" (the default), someone who joins becomes required and -someone who leaves drops out.
    • -
  • Authorization. The asker is stamped by the host (a worker asks only in its own session's scope; secret questions -only from server askers). Each question stores who may answer it; a guest's send access does not imply approval -rights. Secret answers never appear in open, documents or logs.
    • -
  • Link drops and crashes. A rejected ask is not a denial. Askers retry with the same key until their own signal -aborts; a pending ask counts as an active call for keep-alive.
    • -
  • Where it lives. On the host where the asking work runs: session questions on the session's worker, login -questions on the server. UIs reach both through the usual relay.
    • -
-

The store in Chord v2 terms:

-
host.serve(QuestionStore, ({ scope }, self) => {
-  const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed"));
-  const open = self.derive("open", [pending.value.readable], ([all]) => all);   // data only, no resolvers
-  return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) };
-});
-
-host.serve(Questions, ({ scope }, principal, self) => {
-  const store = self.get(QuestionStore, { scope });
-  if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) };
-  if (!mayView(principal, scope)) throw forbidden();
-  return {
-    variant: "viewer",
-    open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)),
-    answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx),
-  };
-});
-

The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and -closed with it). Its close rejects every waiter; askers retry with their key against the new store after a reload.

-
Review: questions.md predates the v2 rules

apps/p2/docs/questions.md writes the store as self.resource(...) with a replace hand-over, which the design -rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with host.root, which Q6 -replaced with chord(fn, context) (chapter XIV). Design §20 says the Questions store "must live in an object (it -predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering -per viewer in the store, once per variant, may be cheaper for large audiences. The ask signature passes typed -objects as arguments, which is fine (data), but the audience readable form cannot cross a link: across links, -audience must be data (principals) or computed by the store.

-
-

Choosing

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
CaseUse
a multi-step login in one ownera flow: the owner's job keeps the resolver, state shows the steps
a tool needs the owner's approvalQuestions, first, audience the owner
the agent asks everyone presentQuestions, all, audience from presence
deploy needs 2 of 3 approvalsQuestions, quorum(2)
a notice (auth URL, device code, progress)not a question: the asking work's own progress state
-
- -
XIII

Extension Points

The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge.

-

The notes plugin wants a download route, GET /s/:session/notes.txt. A logs plugin wants to time every request. -Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these -contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two -things pass: object types and registries.

-

The facet, completed

-

The facet has appeared since chapter II as the place that serves types. It is also itself a pure object with the -host's lifetime:

-
// notes/server.ts
-export default defineFacet((host, self) => {
-  const prefs = self.state(NotesPrefs);                                  // plugin-wide documents
-  self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } });
-  host.serve(NoteCountStore, …, { lifetime: "host" });
-  host.serve(NoteCounts, …);
-});
-
    -
  • It may hold documents, references, routines and contributions; no natives (those need an object with its own -lifetime).
    • -
  • Its identity is the plugin's name, so its persisted documents survive restarts and reloads.
    • -
  • Served objects go live after the facet and may read and write its documents (chapter V).
    • -
  • host.serve and self.contribute are valid only while the setup runs. The facet never self.gets a type it -serves.
    • -
-

Registries

-

A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor -contributes at most one value per registry; the table is an immutable, ordered list of (contributor, value); -the registry's compose turns it into the value consumers use.

-
// owned by the HTTP component
-export const Http = defineRegistry<HttpContribution, Router>("chord.http", {
-  version: 2,
-  accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) },
-  compose: (table, { report }) => buildRouter(table, report),   // pure; always delivers a value
-});
-
-self.contribute(Http, { routes: { … }, wraps: [ … ] });          // a contributor
-const router = host.registry(Http).value;                        // a consumer: live
-

compose runs at runtime, whenever the table changes: after a reload's commit, when a contributor goes live or -ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native -is open), so no check before a reload could be complete. compose always delivers a value with its fallback built in -(e.g. the first by plugin name wins) and reports conflicts through report, which become owner notices and telemetry -events. Readers get a plain value and never handle conflicts. Only if compose throws is the registry in an error -state, and readers get an error when they use it. Work that started earlier keeps the value it read.

-

Rules:

-
    -
  • Who contributes (R21): only facets and lifetime: "host" objects; only their self has contribute (a type -error elsewhere). Every object may read registries.
    • -
  • A contribution is in the table exactly while its contributor is live: a facet's while its plugin is loaded; an -object's from live until it ends, fails or restarts.
    • -
  • Order is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. -Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: keyed(table, entriesOf), -ordered(table, itemsOf, orderOf?).
    • -
  • One authoritative declaration (Q8): only the defining plugin's facet declares the registry to the host -(compose, version, accepts). Other plugins' tokens only identify it by name and types. A reload of the defining -plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the -registry has no value (readers get an error like a failed reference) and contributions wait in the table.
    • -
  • Versions (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted -by the defining facet's pure accepts. One without a matching version or adapter is left out and reported naming -plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and -are not schema-checked.
    • -
  • Use sites guard contributions. A contribution is untrusted code (third-party plugins are a goal). Every use site -catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others.
    • -
  • The owner decides conflicts, order and selection. Registries add behaviour; they never replace an owner's checks.
    • -
-

Natives through registries

-

Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; -consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models -plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload -the next request gets the new runtime, while a running stream finishes on the old one.

-

A handler that stores const r = entry() in a module variable and uses it for days has a bug: after the contributor -restarts it holds a closed native.

-

Contributions that change

-

Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed -for its contributor's life (dynamic parts behind handles read at use) or a readable, which Chord subscribes to; -on every change Chord replaces that entry and runs compose again. For in-process values, self.writable(initial) is -a host-local readable of any value (no JSON, no wire, compared by identity) with set:

-
const ext = self.writable(mcpExtension([]));                   // a Durable Extension, no tools yet
-self.contribute(HarnessExtensions, ext);
-self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools)));
-

There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer -contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension -without tools, empty routes); the table keeps exactly one entry per live contributor.

-

Optional dependencies

-

Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin -manages, but ports is optional. So radius owns radius.forwards and ports contributes its forwards; without ports the -table is empty and radius works.

-

The HTTP component

-

HTTP is an optional Chord component (@earendil-works/chord/next/http) built on a registry. A route gets a plain -request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed -and capped JSON bodies, uploads in chunks) and a ctx whose principal is the request's, so ctx.get(Notes, …) gives -exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges -with 206 and 416, HEAD, download headers with a safe filename, nosniff, CSP and its nonce, caching, streaming, abort -on disconnect):

-
type HttpResponse =
-  | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string }
-  | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise<Uint8Array> }
-  | { kind: "redirect"; to: string } | { kind: "error"; status: number };
-
-// notes/server.ts
-self.contribute(Http, {
-  routes: {
-    "GET /s/:session/notes.txt": async (req, ctx) => {
-      const notes = await ctx.get(Notes, { session: req.params.session });   // the requester's view
-      const text = new TextEncoder().encode(render(notes.notes.value));
-      return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength,
-               read: async (offset, length) => text.subarray(offset, offset + length) };
-    },
-  },
-});
-

Routes compose with keyed: two contributors with one pattern is a reported conflict, and the first by plugin name -wins. Wraps are middleware in phases the owner defines (observe, headers, transform, outermost first), -within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's -lifecycle phases and encapsulation are the model, not Express.

-
// logs/server.ts
-self.contribute(Http, {
-  wraps: [{
-    name: "access-log", phase: "observe", scope: "all",          // default scope "own": only own routes
-    wrap: async (req, next, ctx) => {
-      const t = Date.now();
-      const res = await next(req);
-      log(req, res, Date.now() - t);
-      return res;
-    },
-  }],
-});
-

route narrows a wrap to one route; a route that does not exist is a reported conflict. Wraps may add a security -boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. -Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary.

-

Channels and big bytes

-

Channels are WebSocket byte channels (send, onData, close) for terminals, separate from object links and -with their own acknowledged windows, so a large state message never holds up keystrokes.

-

Big transfers are chunked by the owner, not by Chord: bytes are a schema type (bytes()), carried natively -(never base64), and a 50 MB upload is a protocol of plain calls:

-
beginUpload({ name, size }, ctx)               → { upload }
-append({ upload, offset, bytes }, ctx)         // about 1 MiB per chunk; the awaited call is the backpressure
-finishUpload({ upload }, ctx)                  → { id }
-read({ id, offset, length }, ctx)              → bytes
-

After a link drop the append in flight fails like any call; the writer asks the owner for the stored offset and -resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules -for drops, cancellation, relays and restarts, which this offset protocol already survives.

-
Built so far

Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the -kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the -playground serves its pages with plain app code. Their design stays the target.

-
-
- -
XIV

The Agent

pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people.

-

Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, -whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs -plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable -and shows only how it plugs into Chord.

-

Shared foundations

-

Durable and Chord v2 share modules: Durable imports Chord's context, delta, json, output and sqlite, and v2 -imports the same, so Context, Draft and JsonValue are one type each. A v2 readable is today's -replicatedState (chord/src/state.ts), so Durable's viewState() and document watches can be exposed as Chord state -members without conversion. Only plain data crosses between Durable and Chord objects.

-

The harness is a core object

-
// studio/src/core/harness.ts (base code)
-import { Harness as DurableHarness } from "@earendil-works/pi-durable";
-
-export const ChordHostKey = createContextKey<Host<P>>("studio.chord-host");
-export const HarnessModels = defineRegistry<Models, Models>("studio.harness-models", { version: 1, compose: firstModels });
-export const HarnessEnvironments = defineRegistry<EnvFactory, EnvFactory>("studio.harness-envs", { version: 1, compose: firstEnv });
-export const HarnessExtensions = defineRegistry<Extension, RegistrySnapshotSource>("studio.harness-extensions", {
-  version: 1,
-  compose: (table, { report }) => extensionSnapshot(table, report),      // one Durable Extension per plugin
-});
-const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", {
-  version: 1, compose: onlyEntry,                                        // core-private: one contributor
-});
-
-host.serve(Harness, (_input, self) => {
-  const info = self.get(SessionInfo, {});
-  const harness = self.open(async ({ signal }) => {
-    const { id, dataDir } = info.init.value;
-    const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT));
-    return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), {
-      models: () => host.registry(HarnessModels).value,                  // read at each use
-      env: (target) => host.registry(HarnessEnvironments).value(target),
-      settings: () => currentSettings(),
-      registry: durableRegistry(host.registry(HarnessExtensions)),      // a RegistryReader over the composed value
-    }, base);
-  }, (h) => h.close(BACKGROUND_CONTEXT));
-  self.contribute(HarnessRuntime, () => harness.value);                 // for core's conversation objects
-  return { … };
-}, { lifetime: "host" });
-
    -
  • Lifetime "host": it runs for the worker's whole life, also while nobody holds it, because Durable tasks -continue.
    • -
  • Base code: it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host -restart) does.
    • -
  • Registries, looked up at each use: Durable's models and settings accept functions (pi 7ec1f648e), so each -generation gets the current value. After a models reload the next request uses the new runtime; a running stream -finishes on the old one.
    • -
  • Durable's storage is its own. Chord documents never commit together with Durable transcripts.
    • -
-

The design writes the call as Durable.open(storage, { … }); pi-durable's actual entry point is -Harness.open(storage, options, context), used here as DurableHarness to keep it apart from the core object -Harness.

-

Start order

-

At host start Chord loads every facet and waits until every facet and every lifetime: "host" object other than the -harness is live or failed: the start barrier. Then the registries hold the contributions of every plugin that -loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model -runtime, is therefore lifetime: "host". Nothing waits for specific contributions; what is missing shows where it is -used:

- - - - - - - - - - - - - - - - - - - - - - - -
MissingWhat happens
a toolits call gets an error result the model sees
a modelthe generation reports "model unavailable" and waits, not fails, until a model appears
a task definitionthe task stays blocked (missing_task) until the plugin is installed or the task is aborted
a plugin that failedreported with its error; a facet in backoff contributes later and the registries recompose
-

The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and -hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must -show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then.

-
Review: the harness exception to the barrier (C17, R17)

R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: -no harness-specific exception; the harness is an ordinary object the app's start code holds after load resolves. -Not decided. chord-next-walkthrough.md concept 28 still says the harness's activation is the start barrier, which -R17 replaced.

-
-

Tools that call Chord

-

Durable tools, hooks and sections receive Durable's ToolExecutionApi (or HookApi) and a Context, never a Chord -ctx (Q6). Core put the host into the harness's base context under ChordHostKey, and every phase context inherits -it. A core helper opens a Chord call scope from any Durable context:

-
// studio/src/core/chord.ts
-export function chord<T>(context: Context, fn: (ctx: Ctx) => Promise<T>): Promise<T> {
-  const host = context.value(ChordHostKey);
-  if (!host) throw new Error("not inside the Studio harness");
-  return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry });
-}
-

The scope runs as the host principal; attribution is data; references taken with ctx.get are released when fn -settles. There is no wrapping of tools and no app-specific defineTool. The notes plugin gives the agent a tool:

-
// notes/worker.ts
-const addNote = defineTool({
-  name: "notes_add",                                         // tools are named <plugin>_<name>
-  description: "Add a note to this session's notes panel",
-  parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }),
-  execute: (args, api, context) => chord(async (ctx) => {
-    const info = await ctx.get(SessionInfo, {});
-    const store = await ctx.get(NoteStore, { session: info.init.value.id });
-    const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx);
-    return { output: [{ type: "text", text: `added note ${id}` }] };
-  }, context),
-});
-
-export default defineFacet((host, self) => {
-  self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] }));
-  …
-});
-

A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so -a tool call already running finishes on the old code and the next phase uses the new one.

-

Tools that need a person

-

Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's -state: the Questions service (chapter XII). The model-facing ask_user tool and an approval hook both call -Questions.ask; tabs render the open questions and answer them.

-
hook(ToolTask, {
-  beforeTool: async (call, api, context) => {
-    if (!needsApproval(call)) return undefined;
-    const decided = await api.memo<boolean>("approval", context);       // a rerun does not ask again
-    if (decided !== undefined) return decided ? undefined : { block: "Denied" };
-    const allowed = await chord(async (ctx) => {
-      const q = await ctx.get(Questions, { scope: { session } });
-      for (;;) {
-        try {
-          const r = await q.ask(
-            { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" },
-            { audience: approvers, policy: { mode: "first" }, key: api.taskId },
-            ctx,
-          );
-          return r.answer;
-        } catch (error) {
-          if (ctx.signal.aborted) throw error;                          // a link blip is not a denial: retry
-          await backoff(ctx);
-        }
-      }
-    }, context);
-    await api.memo("approval", allowed, context);
-    return allowed ? undefined : { block: "Denied by the user" };
-  },
-});
-

Durable runs beforeTool in the tool task's call phase; after the execute checkpoint a crash retries execution -without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the -approval hook after hooks that rewrite arguments.

-

Conversations for the UI

-

Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 -design; one construction that follows the rules: a core ConversationStore { conversation } (local) opens the Durable -watch as its own native, getting the Durable harness from the core-private HarnessRuntime registry contribution (a -native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal -Conversation { session, conversation } views in front of it check who may send (chapter XIX).

-

Code mode

-

Code mode is a plugin. It owns its pi-codemode sandbox as a native; its tool entry looks up the current sandbox -when execute() starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the -new sandbox. On reload and stop the old sandbox is sealed and drained in close: running scripts finish or hit -their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today.

-

What lives where

- - - - - - - - - - - - - - - - - - - -
DataWhere
transcript-bound: entries, documents that fork and rewind with a conversationDurable
app state: session list, notes, drafts, grants, settingsChord documents
data core enforces on conversations (read-only one-shots, cwd followers)a core per-conversation policy document, written when the conversation is created (createConversation({ init }))
-

Telemetry: pass the host's adapter as HarnessOptions.telemetry; Durable stores trace carriers on submissions and -tasks (chapter XVII).

-
- -
XV

Reload and Rollout

What happens when you press Reload: per host, per object, and across the server, the workers and every tab.

-

Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters -asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in -documents) exists so that this moment is safe. This chapter shows what Chord does with it.

-

On one host

-
1. load      new bundles; an import or syntax error rejects
-2. staging   facet setups and every live object's new factory run with a recording self
-             (declarations only, nothing activated); any throw rejects
-3. commit    the host switches to the new code atomically
-4. after     per object: pure objects swap, native objects restart
-

Rollback is all-or-nothing per host. Any load or staging error leaves the whole host on its previous code, so a -host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed -open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory -trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with -contract_mismatch and a notice naming plugin and type.

-
Reloading @studio/notes on worker S1, object by object

Load and stage

The new bundles load. The facet setup and the factories of Notes (three views), NoteStore { S1 } and NoteInbox -run with a recording self: they declare, nothing activates. Nothing throws, so the host commits to the new code.

-

NoteStore: pure, swapped in place

NoteStore has no self.open. Its new methods and derives take over; NoteList, the drafts, its references and the -running import job stay. Routines restart, serialized: the new cleanup waits for the old handler in flight. The gate -opens.

-

Running calls finish on old code

An add that was running when the swap happened finishes on the old code with its old handles; its transaction still -commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that -touches the migrated document throws, the document stays intact, and that call fails once.

-

The import job keeps its old run

The job's run keeps the code it started with. When it settles, the new finish records the outcome.

-

The views: pure, swapped

The three Notes views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with -the new sort and the difference is sent. Nobody sees a reconnect.

-

NoteInbox: native, restarts

NoteInbox has a watcher. Its running work ends aborted, the watcher closes with reason reload, the new factory -runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the -watcher's business: Studio's watchDir scans the directory once when it opens.

-
-

Why natives restart instead of being handed over: a native has no line, no transactions and no schema. If old and -new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base -code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild -in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket -(relayed connections drop) hurts.

-

Removing a type: its objects end (routines stop, natives close, references end with an error); in-memory documents -go with them; persisted ones stay until retired. Consumers elsewhere see a restarting object like any object that -briefly goes away: loading, parked calls, then live.

-

UI facets reload too. A reload unmounts and remounts components. That is why "leaving" (abort a login, close an -overlay) is never derived from an unmount (chapter VIII).

-

Across hosts

-

A revision is (server incarnation, sequence) and names the complete desired code set: every facet with its -digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it -runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), -so a server restart with unchanged code triggers no reinstall.

-

The order is server, workers, clients. Each host reports installed, installed_failed (with the objects that -failed) or rejected (with the reason; it stays on its previous code). A server that rejects stops the rollout. A -host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized -per host and bounded (60 s).

-

Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response -carries the provider's revision and status:

- - - - - - - - - - - - - - - - - - - -
ProviderThe call
ahead, or behind and installingupdating: values readable, calls park (with the park deadline)
rejectedfails at once; retried on the next revision
same revisioncontract_mismatch: a real mismatch, final
-

A host checks its own install status first: if its own install was rejected, a call fails at once with the reason -instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. -There are no cross-host transactions and no votes.

-
tabrevision 8serverrevision 8worker S1installing 8install revision 8call Notes.add (new contract)relayupdating (rev 7, installing)updatingpark (deadline 60 s)installed 8provider at 8call Notes.add (resent from park)relayresult
Figure 9. A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision.
-

Start barrier

-

At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start -data among them), wait until every facet and lifetime: "host" object other than the harness is live or failed, then -resume dependent work. Plugin reloads need no barrier.

-

Loading and the dev loop

-
    -
  • Roots: plugins come from built-in, global (<data>/plugins), package (npm or git, installed with ---ignore-scripts) and project roots, found by directory listing; no source parsing. The server selects which load. -Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects <plugin>.<name>, -tools <plugin>_<name>.
    • -
  • Loader D: one base bundle per surface (host packages and the app runtime) plus one script per facet, with host -packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per -(facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static -imports (tests, Durable Objects) work the same, with the app passing the plugin name to load.
    • -
  • Dev loop: a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web -shell, a CLI, an agent tool such as p2's reload_plugins) reloads the changed plugins, or restarts all hosts for -base code. Reloads are always explicit.
    • -
-
Review: project plugins (#14, deferred)

Per-directory plugins with trust prompts and snapshots (apps/p2/docs/plugin-loading.md, still the reference for -roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not -settled.

-
-
Built so far

Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and -a development freeze, both removed from the design.

-
-
- -
XVI

Changing Contracts

How a contract gets a new version while old tabs, old workers and old plugins are still talking to it.

-

Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout -finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's -web bundle. A plugin from another author was built against Notes version 1 and is not updated yet. In both, a -consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is -also the one part of Chord v2 that is built: everything in the sections on evolve, checkAccepts and fingerprints -runs today in packages/chord/src/next/contracts/.

-

What has a version

-

Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SurfaceVersionWhen an old one shows upChapter
object type (contract)version plus fingerprintevolve and checkAccepts on the provider; otherwise contract_mismatchthis one
documentversion plus schema fingerprintpure migrate on access; a schema change without a bump fails the object brokenV
jobversionaccepts on outcomes and running; blocked without an adapterXI
registryversionaccepts adapts contributions; one without a match is left out and reportedXIII
nativenonethe object restarts on reloadIX
Durable tasks and documentsDurable's versionDurable's migrateXIV
link protocolprotocol versionrefused; hosts update together, a web page reloadsXVII
UI components, Chord and app APIsnone yetexposed as they are; an SDK will version them later
-

The rule behind every row is the same as for documents: every change bumps the version, additive ones included. -Chord never guesses compatibility from the shape of two schemas.

-

A history, not a file of versions

-

A contract history starts with a base: a full remote() or local() declaration with an explicit version. The -Notes contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a pinned -flag and editors get a pin method. Instead of editing the declaration, the plugin continues the history:

-
// notes/contract.ts
-import { bool, evolve, nul } from "@earendil-works/chord/next";
-
-export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() });
-export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() });
-
-export const NotesV2 = evolve(Notes, (e) =>
-  e.state("editor", "notes", obj({ items: arr(NoteV2) }))
-    .state("reader", "notes", obj({ items: arr(PublicNoteV2) }))
-    .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()),
-);
-

evolve(prev, (e) => …) appends one version to prev's history: NotesV2.version is 2, and NotesV2 still knows -version 1 exactly as it was declared. Every version is materialized and fingerprinted at declaration. That is what -makes old code safe: an old plugin bundle that contains its own copy of Notes version 1 computes the same -fingerprint as the history inside NotesV2. There is no build step, no generated file and no fingerprint copied by -hand.

-

The edits:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
EditChangesKind
e.state(variant, member, schema)one state member of one variantremote
e.addState(variant, member, schema) / e.removeState(variant, member)adds or removes a state memberremote
e.state(member, schema), e.addState(member, schema), e.removeState(member)the same, flatlocal
e.args(name, schema) / e.result(name, schema)one side of a methodboth
e.method(name, args, result)both sides of a methodboth
e.addMethod(variant, name, args, result)adds a method (or an existing one to another variant, same signature)remote
e.addMethod(name, args, result)adds a methodlocal
e.removeMethod(name)removes a method from every variantboth
-

Edits chain. The method(...) wrapper appears only in base declarations; edits take the schemas directly, so a -method without a result is added with nul(). The rules of chapter III still hold across versions: a method has one -signature in every variant, and a member's kind never changes, not even after it was removed and added again.

-

What evolve cannot do: change the input or the set of variants. Both change what an object is (its identity, -who gets which members), and no adapter can turn one into the other. That needs a new history base, below.

-

The provider says what it accepts

-

NotesV2 says what changed. It does not say how version 1 consumers are served; the provider decides that, with -mappings that live in its own code, never in the contract. The worker facet declares them with checkAccepts:

-
// notes/worker.ts
-import { checkAccepts } from "@earendil-works/chord/next";
-
-const toV1 = ({ pinned, ...note }: Static<typeof NoteV2>) => note;
-const toPublicV1 = ({ pinned, ...note }: Static<typeof PublicNoteV2>) => note;
-
-const notesAccepts = checkAccepts(NotesV2, {
-  1: {
-    state: {
-      editor: { notes: (current) => ({ items: current.items.map(toV1) }) },
-      reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) },
-    },
-    methods: {},                                   // add and remove did not change; pin is new
-  },
-});
-// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }]
-
-host.serve(NotesV2, notesFactory, { accepts: notesAccepts });    // working name for the objects-layer option
-

checkAccepts is pure: it checks the mappings without calling them and returns only the version metadata of the -old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later -compares a consumer's version and fingerprint against this list.

-

The mappings point in the direction the data flows:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Member in the old versionMappingDirection
state, changed(currentMember) => oldMemberprovider's current value to what the old consumer expects
state, removed(currentVariantState) => oldMember, or "refuse"receives the whole current state of that variant
method arguments, changedargs: (old) => currentold caller's arguments to current ones
method result, changedresult: (current) => oldcurrent result back to the old shape
method, changed or removed{ call: async (oldArgs, ctx) => oldResult }, or "refuse"replaces the method for old callers
member added sincenothingan old consumer never asks for it
-

A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one -can be computed from what the variant has now, or refused. e.method(name, args, result) replaces both sides and -therefore needs both mappings, even if one side stayed the same (then an identity mapping).

-

Required, at type level and again at runtime. For every member that exists in the old version and changed or was -removed since, the mapping is required by the type of checkAccepts. Because plugins are bundled without type -checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version -or a wrong shape throws, naming the member's path (for example accepts[1].state.reader.notes). An absent state or -methods section counts as empty and may be left out only when nothing there needs a mapping.

-

Conservative with no-op edits. An edit that sets a schema to what it already was, or changes a member and then -changes it back, still requires an identity mapping. TypeScript erases constraints such as { max: 2000 } from the -static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell.

-

Old versions map straight to the current one

-

Studio later replaces remove with archive, which keeps the note in a trash list:

-
export const NotesV3 = evolve(NotesV2, (e) =>
-  e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()),
-);
-
-const notesAccepts = checkAccepts(NotesV3, {
-  1: {
-    state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } },
-    methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } },
-  },
-  2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } },
-});
-

Each entry maps one old version directly to the current one, not through the versions in between. Version 1 -needs the state mappings (its notes have no pinned) and the remove replacement; version 2 needs only the -remove replacement, and leaves state out because its state is unchanged. A member that an old version never had -needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is -deleting its entry: version 1 consumers then get contract_mismatch.

-

Binding an old consumer

-

A hold carries only the consumer's contract version and fingerprint, never a description of the contract (R6). -The provider decides with its own schemas:

-
THE HOLD ARRIVESOUTCOMEholdversion 1, fingerprint F1equal to current?version and fingerprintaccepted old version?F1 from the provider's own historybindcurrent membersbind adaptedthrough the mappingscontract_mismatchor updating, during a rolloutyesnoyesnoA mismatch fails only that consumer's binding. The provider object and its other bindings are not affected.
Figure 10. What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input.
-

Once a version 1 tab is bound, it is served as if version 1 were current:

-
A version 1 tab on a version 3 worker

Bind

The tab holds Notes { S1 } with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and -notesAccepts lists version 1, so the hold binds adapted. The tab gets the editor variant as version 1 defined it: -notes, add, remove. It never sees pin or archive.

-

State through the mapping

Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value -and keeps the result per binding. The snapshot the tab receives has no pinned fields.

-

A commit

The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, -Chord applies the mapping to the new value and sends the difference to its last adapted value (diffRevisions). The -mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent -whole on every commit.

-

A call

The tab calls remove({ id }). Its arguments are checked against the provider's own version 1 schema, then the -call replacement runs archiveNote on version 3 code. Its null result is what version 1 expects. Had the method -only changed, args would have mapped the arguments forward and result the result back.

-

The worker moves on

A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), -the hold gets contract_mismatch, and Studio's panel shows "reload to update" instead of failing silently.

-
-

A new history base

-

Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from -{ session } to { project }. That is a new base, written in full, with an explicit version that continues the -published numbering:

-
export const NotesV4 = remote("notes", {
-  version: 4,                                      // explicit: the published history used 1 to 3
-  input: obj({ project: str() }),
-  placement: () => Server,
-  variants: {
-    editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } },
-    reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } },
-  },
-});
-// checkAccepts(NotesV4).versions is empty
-

A new base has no history before its version and cannot accept older versions, not even for an additive change. -Every consumer built against versions 1 to 3 gets contract_mismatch and must update. Later evolve calls continue -from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or -copy its fingerprint; the contract objects are branded and evolve and checkAccepts reject copies.

-

Local contracts

-

local contracts (chapter IV) evolve the same way, with a flat members map and no variants:

-
export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2));
-checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } });
-

Same-host consumers of a local type are usually reloaded together with their provider, but not always: another -plugin on the same host may be built against an older version. The rules are identical; only the variant argument is -missing.

-

Implementing a contract

-

Implementation<typeof NotesV2> is the type of everything a provider must supply for the current version: one state -getter per member and variant, and one asynchronous method per method, each receiving (args, ctx):

-
const impl: Implementation<typeof NotesV2> = {
-  state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } },
-  methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) },
-};
-

Keep declaration types inferred (no annotation on NotesV2): the inferred type carries the complete history, and -checkAccepts needs it to know what changed.

-
Review: L1's implementation shape and the factory's return value

Implementation<C> in src/next/contracts/types.ts is { state: { variant: { member: getter } }, methods }. The -design's factories (design.md §6, used throughout this guide) return { variant, member: value, method: fn }. -The objects layer (L4) has to connect the two, and also define how serve receives the checked accepts (this guide -uses the working name { accepts } as a third argument) and what "refuse" does at runtime. The natural reading, -used in the stepper above, is that a refused method call and a hold that needs a refused state member end with -contract_mismatch; no source says so.

-
-

The fingerprint

-

The fingerprint is a 64-bit hash (hash64, portable across runtimes) of a normal form of the contract's -schemas. The normal form decides what counts as a change:

- - - - - - - - - - - - - - - - - - - -
IgnoredSignificant
order of object keys, required and enum (sorted)order of union and intersection branches (anyOf, oneOf, allOf) and of tuples
annotations on schema nodes: title, description, examples, $comment, deprecateddefaults, formats, patterns, bounds, $ref names
placementinput, variants, member kinds and every member schema
-

Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. -Annotations are dropped only on schema nodes; an object property that happens to be named title is data and counts.

-

Only data schemas are allowed. One walker copies, validates and normalizes each schema against a whitelist of -TypeBox keywords: primitives, arrays and tuples, objects and records, unions and intersections, $defs and $ref -(use $ref for recursive types), constraints and annotations. Conditional and dependent schemas, not, -definitions, contentSchema, unevaluated*, contains, propertyNames, unknown keywords, executable or custom -TypeBox features, arbitrary Unsafe and refinements, accessors, symbols and cyclic objects are rejected at -declaration. Bytes are the one special leaf: bytes({ min, max }) checks a Uint8Array and has the data marker -{ type: "bytes", minLength?, maxLength? }, so it hashes like any schema; TypeBox's checking still works on it.

-

Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. -NORMAL_FORM_VERSION is part of every digest, so a change of the normal form is a link protocol change.

-

Not a security boundary. A fingerprint says "we were compiled against the same contract"; it proves nothing. A -hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own -schemas (chapter IV), whatever the fingerprint says.

-

Rules of thumb

-
    -
  • Treat a published contract like a published API: every change is evolve, every evolve needs a decision about -old consumers.
    • -
  • Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over -the value, not a query.
    • -
  • Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the -metrics show no old bindings.
    • -
  • Change the input or the variants only with a new base, and only when updating every consumer is acceptable.
    • -
  • Contract, document and job versions are separate and each is decided on its own. Pinned notes needed NotesV2 -for the wire and NoteList version 2 with a migrate that adds pinned: false (chapter V), but no new -ImportJob version, because its outcome did not change.
    • -
-
Built so far

This chapter's contract layer is built: packages/chord/src/next/contracts/ (commit 7d2bb8ea8, about 1,300 lines -with remote, local, method, evolve, checkAccepts, Implementation, the schema helpers, fingerprintOf, -schemaNormalForm and NORMAL_FORM_VERSION), with nine test files covering evolution, fingerprints, the hash, -local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no -mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to -the objects and protocol layers, not built yet.

-
-
- -
XVII

On the Wire

What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces.

-

Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord -moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when -you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and -no callbacks: each of those rules removes a cost on some hop.

-

What a state member sends

-

Every exposed document, derive or readable is a state member. Over a hold it produces one of these:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
UpdateWhen
snapon hold and on re-hold: the whole value, once
opsper commit: the draft's recorded ops, no diff
ops from a diffper change of a derive (and of an adapted binding, chapter XVI)
throttled opsjob progress
reseta migration commit replaced the value (adapted for older consumers)
errorthe member failed
-

Commits are atomic for consumers. On the same host, subscribers are notified once, after every document of the -commit is adopted. Remotely, one state message per hold carries a batch of member updates that is applied before -listeners run, and a commit that touches several holds on one link travels as one state message on that link. -A consumer never sees half a commit.

-

Messages

-

A message is a header plus opaque parts (R1). There is one part per hold the message concerns; the header says -which hold each part belongs to and how large it is.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
DirectionKindParts
consumer to providerhold, release, call, cancelcall has one (the arguments)
provider to consumerboundone: the snapshot
stateone per hold the commit touched on this link
resultone
gonenone
-

There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link.

-

One binary format on every link (Q4): u32 header length | header JSON | part bytes, as one Uint8Array. A -WebSocket sends it as a binary frame, IPC passes it with serialization: "advanced", a memory link delivers it -asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol -layer; link adapters only move bytes. Tests on memory links therefore run the production path.

-

A part is encoded once, at its origin, with that hold's own Delta encoder (paths are interned per hold, end to -end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, -whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec.

-

The header is a TypeBox schema (R4), a union tagged by kind; the protocol types derive from it. Headers from -untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and -workers) are not checked. Unknown ids are ignored on both sides (Q1). They are normal races: a result crossing a -cancel, a call crossing a gone, a state part crossing a release. The side that ended a hold or call settles -its own pending work: a consumer that receives gone(h) rejects its pending calls on h, and the order per -direction guarantees nothing else for h follows. Only a header that fails the schema is a protocol error, and it -closes that link.

-

The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get -the server's protocol.

-

Through a relay

-

A tab never talks to a worker directly (chapter I). The server relays, and a relay reads and rewrites only -headers: it maps hold and call ids between its two links, stamps the principal on hold and call (a client's -principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination -link, so each link gets only its own holds' parts. It never parses a payload.

-
owner's tabhold h1Ann's tabhold h2serverrelayworker S1NoteStore, Notescall c7 Notes.add (1 part)map ids, stamp ownercall c3 (part forwarded as is)method runs, self.tx commitsstate: parts for h1', h2'group parts per linkstate (h1 part)state (h2 part)result c3result c7
Figure 11. The owner adds a note while Ann watches. The commit leaves the worker as one state message with two parts; the server splits it by destination without decoding either part.
-

The state message reaches the owner's tab before the result, so when await notes.add(…) resolves, the new note -is already in notes.value. Ann's view is a different variant (reader) and gets its own part from its own derive; the -relay neither knows nor cares.

-

Flow control

-

Each link has one byte window, counted from each message's byteLength (parts are already encoded bytes; nothing -is measured or stringified to count it). The window covers every message, first snapshots included.

-
    -
  • Snapshots are always admitted (R19), even when larger than the window; they use it up. Whether a hold -overflows is decided on the changes queued after the snapshot.
    • -
  • Under backpressure a commit's ops are collected per hold and encoded into one part when credits return. The -consumer still gets the commit atomically.
    • -
  • After an overflow the hold is dropped and established again once the previous snapshot's debt has drained -(credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot -in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window.
    • -
  • Calls are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. -Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation -protect them.
    • -
  • Byte-heavy traffic such as terminals uses separate channels (chapter XIII), so a large state message never -holds up keystrokes.
    • -
-

Recovery: after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in -flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and -re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the -per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything.

-

Size limits

-

Limits apply per part (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the -header before anything is decoded; zlib's maxOutputLength enforces the decoded limit in the one decode function.

- - - - - - - - - - - - - - - - - - - -
SituationWhat fails
a part over the limitonly its own hold or call, with too_large; the other parts of the message apply
a snapshot over the limitits hold ends failed, with no automatic re-hold
a WebSocket frame over maxPayload (e.g. 64 MiB)the connection closes (code 1009); this is the transport's hard cap
-

Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The -decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission -(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own.

-

Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP -component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII).

-

What each hop costs

-

Chord v2's goal is as little compute and memory as possible while staying correct: O(size of the change) per -commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new -viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives -are almost free: diffRevisions on shared subtrees takes 0.018 ms, applyImmutable of a small change 0.004 ms.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
HopWork
commitDelta's prepare (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta
reads in a transactionthe draft overlay, no copy; an unchanged draft writes nothing
subscribers on the same hostthe same value and ops, by reference
derivescompute plus diffRevisions; unchanged parts stay the same objects (mapShared, chapter VIII)
sendone encoding per hold part at its origin, size taken from the encoded bytes
relayheader only: ids mapped, principal stamped, parts grouped; parts never parsed
receivecompiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized
applyapplyImmutable, structure shared
new holdthe value encoded once (unavoidable)
memoryone authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding
-

These are targets. The frozen kernel measured far worse, which is why the design is so strict here:

-
35 msone small change to a 500 KB value in the frozen kernel, no viewer
+13 msper viewer on top (10 viewers: 130 ms)
17 / 33 / 101 msa tiny SQLite commit after 0 / 10 / 50 earlier deltas
-

Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, -JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and -a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes -0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the -before/after per hop is in apps/p2/docs/p3-reviews/v2-cost-model.md.

-
Review: v2-cost-model.md is partly outdated

The measurements and the per-hop "after" targets in apps/p2/docs/p3-reviews/v2-cost-model.md are current. Its -closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions -once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), -and §10 still describes the description fetch.

-
-

Cost guards

-

Chord reports full-value walks, deep copies, encodes, schema validations and materializations as ordinary -telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the -in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers -does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A -vitest bench suite measures the hops.

-

Ways to make it slow anyway

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MistakeWhy it costsInstead
a derive that builds new objects for unchanged dataevery commit diffs and sends O(value)keep unchanged items identical: mapShared, previous (chapter VIII)
one long array that changes often (R24)sharing is per path; each change copies one slot per element and visits eachkeyed documents or a bounded window
bytes as base64 stringsinflated, parsed, compressed as textbytes() leaves; chunked transfers for files
awaiting I/O inside self.txthe object's line is blocked for every callerdo the I/O first, then a short transaction
storing registry entries or references in documentsstale after a reload, not datalook up the current value at use
opening natives at module level or in a facetno owner: never closed, never restartedself.open in an object (chapter IX)
callbacks as argumentscannot cross a link, keep old code aliveregistries, jobs, flows
one big object that everybody viewsone large snapshot per viewer and per reconnectsplit by viewer need; views of the same variant share one value
-

Telemetry

-

Chord reports traces, events and metrics through @earendil-works/pi-telemetry. Each host gets one adapter, -createHost({ telemetry }), NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts -hold the adapter, to start root spans. Code sees spans:

-
    -
  • ctx.telemetry is the current span (Ctx extends Context, so it is the same field Durable and pi-ai read). Every -span carries its adapter's metrics and propagation, so code with a ctx can record a metric or inject a carrier.
    • -
  • ctx.trace(name, attributes, fn) runs fn in a child span with a child ctx.
    • -
  • App values such as the principal stay out of Durable's context.
    • -
-

Across links, call and hold carry the caller's trace carrier as a bounded header field (R5: limited key count -and string lengths); the provider runs the method under it, and each relay adds a chord.relay span. The call from the -sequence above, as a trace:

-
chord.call Notes.add            [owner's tab]
-└─ chord.relay                  [server]
-   └─ chord.method Notes.add    [session:S1] principal=owner
-      · event chord.commit docs=2 ops=5
-

Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from -another process are checked (keys present, strings), and a bad one falls back to a new root.

-

Shared work (one activation serving several gets, a worker start) is its own root span, linked to the trigger -that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background -closes are root spans too, linked to their trigger where there is one.

- - - - - - - - - - - - - - - - - - - -
TierWhat
spanscalls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs
eventscommits inside methods, retries, parking
metricsstate messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards
-

Ids are attributes of spans, never of metrics. Nothing is measured or walked for telemetry; only sizes the code -already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an -error. Chord ships a typed telemetry schema, as pi-ai (AI_TELEMETRY_SCHEMA) and Durable -(DURABLE_TELEMETRY_SCHEMA) do.

-

Storing and reading: @earendil-works/chord/telemetry-sqlite over chord/sqlite (Node and Cloudflare) stores one -row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every -second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface TelemetryStore is shared: -findTraces (by name, attributes, errors, time window, with limit and cursor) returns complete traces, getTrace -one, findMetrics metrics. One renderer, pi-telemetry/render, prints traces as above; it prints links and never -follows them. The playground's trace viewer reads the same store.

-

Durable already emits submit and phase spans, stores carriers on submissions and tasks (TaskOptions.trace), -and pi-ai emits pi.ai.request and pi.ai.poll. With the harness given the host's adapter (chapter XIV), an agent -tool that calls Notes.add through chord(fn, context) shows up as one tree from the model request down to the -commit.

-
Built so far

Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in -Chord is L11. Built in pi: pi-telemetry (propagation, links, metrics, the store interface, the renderer), -chord/context with Context.telemetry, pi-ai and Durable telemetry, and chord/telemetry-sqlite. Delta is built; -mapShared and bytes as atomic leaves in diffRevisions are still to do. The frozen kernel sends descriptions on the -wire and closes a link on an unknown id; both are replaced.

-
-
- -
XVIII

Building p2: the Core

From a list of what the app must do to a set of owners: the base code every p3 plugin builds on.

-

The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the -next two assemble p3, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in -apps/p3, fully isolated with its own data directory, and without migrating p2's data (no import of -catalog.sqlite, shares.sqlite or the subagents table). Its plugins are derived from what the app must do, not -ported one to one from p2's plugin layout.

-
Review: p3 is not designed yet

The design fixes the rules (design.md §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the -per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, -derived from the functionality map in apps/p2/docs/p3-functions/. Object names are working names. Where a domain -has an open decision, the text says which.

-
-

Start from functions, not from plugins

-

The functionality map in apps/p2/docs/p3-functions/ lists what p2 does, in plain words, with four facts per -function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups -the functions into 22 domains:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#Domain#Domain
1sessions12access
2session workers13models, credentials, settings
3conversations and agent14prompt resources
4shell commands15status and presence
5chat mode16search
6agents (subagents, one-shots, peer messages)17remote access and forwards
7code mode18logs and notices
8session files19dev and reload
9environments and execution20web shell
10workspace21TUI shell
11terminals22transport
-

The four facts answer the four questions every Chord object needs, in order:

-
THE MAP SAYSCHORD NEEDSdatawhat, how bigdurabilityephemeral · until restart· persisted ·transcript-boundwhoowner · guest read · guestwrite · workerwhereserver · worker · web ·TUI · environmentthe ownersmallest object it belongstowhere it is keptDurable, persisted orin-memory document, tabvariantsand revocationhost and codeplacement; core or pluginDurability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with astable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client.
Figure 12. From a function in the map to code. Each fact in the map answers one question Chord needs answered.
-

One owner per thing, the smallest object it belongs to. A guest's grant belongs to the access owner, not to the -session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and -rules set at creation are written in the creating commit (Durable's createConversation({ init })).

-

Core or plugin. Core owners are what the app always needs and everything builds on; they are always present, so -other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature -does so through a registry it owns itself (chapter XIII). The risk to avoid is a core god object: each core owner -owns only what belongs to it.

-

The ownership map

-

The result of that procedure for p3, as a starting point:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
DomainOwner object(s)HostCode
accessAccess (grants, devices, share links), authenticateservercore
sessionsCatalog (session records), lifecycle jobsservercore
session workersthe supervisor (route(Session) + idle stop)servercore (app code)
session locationSessionLocation { session }workercore
conversations and agentHarness, ConversationStore, Conversation viewsworkercore
status and presenceSessionStatus (server copy), reporters on workersbothcore
shell commandsShellJob in the chat pluginworkerplugin
chat modechat pluginworkerplugin
agentssubagents plugin (Durable child conversations)workerplugin
code modecode mode plugin (sandbox native)workerplugin
session filesSessionFiles { session } (proposed, #17)servercore or plugin, open
environmentsenvs plugin (config, SSH pools #20)server and workerplugin
workspaceworkspace pluginworkerplugin
terminalsterminal host process (#23), terminals pluginseparate processplugin
models, credentialsmodels plugin, login job + flowserverplugin
settingsSettings (single writer, #21)servercore
prompt resourcesprompt plugin (watchers in the session's environment)workerplugin
searchsearch plugin (index on the server, feed on workers)bothplugin
remote access, forwardsradius plugin, ports pluginserverplugin
logs and noticesNotices (one channel), log captureservercore
dev and reloadthe reload control, reload_plugins toolserverplugin
web and TUI shellsUI facets over the objects aboveclientsshells
-

The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins.

-

The base code

-
apps/p3/
-  src/
-    hosts.ts           Server, Session host kinds
-    principal.ts       P, Revocation, principalKey, revokedBy, can()
-    server/main.ts     createHost, load, route(Session), WebSocket and Radius entry points, supervisor
-    worker/main.ts     createHost with the start data, connect to the parent
-    core/              the core facets: Access, Catalog, SessionStatus, Settings, Notices (server);
-                       SessionInfo, SessionLocation, Harness, ConversationStore (worker)
-  plugins/             chat, subagents, codemode, envs, workspace, terminals, models, login,
-                       prompt, search, ports, radius, files, dev
-

hosts.ts, principal.ts, the SessionInfo start data and the Harness object are the ones the guide already -built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in -plugins/ reloads.

-

Who is calling: authentication

-

Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by -authenticate(req) when a WebSocket connects (chapter III):

- - - - - - - - - - - - - - - - - - - - - - - - - - - -
Way inPrincipal
loopback without a cookie (TUI, CLI, first page load){ kind: "owner", access: "local" }
loopback owner browser cookie (a per-port secret file){ kind: "owner", access: "local" }
a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token{ kind: "owner", access: "device", device }
a guest cookie from a share link{ kind: "guest", session, mode, grant, revision }
the Radius relay channel{ kind: "owner", access: "relay" }
-

The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from -another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, -single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket -before Chord sees it.

-

One permission matrix. Every function in the map names who may use it, and p2 checks those rules in several -places. p3 writes them once, in principal.ts, and every factory and authorize uses it:

-
// src/principal.ts
-type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage";
-
-export function can(p: P, action: Action, session?: string): boolean {
-  if (p.kind === "owner" || p.kind === "server") return true;
-  if (p.kind === "worker") return action !== "manage" && p.session === session;
-  if (p.kind !== "guest" || p.session !== session) return false;
-  if (action === "terminal" || action === "manage") return false;     // terminals: owner only
-  if (p.mode === "read") return action === "read";                     // no workspace for guest read
-  return true;                                                         // guest write: send, upload, stop, workspace
-}
-

Factories pick variants from it (can(p, "send", session) ? "writer" : "reader"), so the matrix is visible in one -file and the members a caller lacks are simply not there.

- -

Access is the core owner of everything that lets someone in: share links per session and mode, guest records, the -device list and the signing key. It lives on the server as the chapter IV pattern: a local store that persists the -documents (only stable identities may), and a remote view for the owner's settings screen:

-
// src/core/access.ts (server, in the core server facet)
-const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true,
-  initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 });
-const Guests = defineState("access.guests", { … });         // same shape: map(Guest), persisted
-const Devices = defineState("access.devices", { … });       // map(Device), persisted
-
-host.serve(AccessStore, (_input, self) => {                 // local: may persist, acts for the host
-  const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices);
-  return {
-    links, guests, devices,
-    removeGuest: async ({ grant }, ctx) => {
-      await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx);
-      host.revoke({ grant });                                 // live access ends now
-    },
-    resetLink: async ({ session, mode }, ctx) => {
-      const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx);
-      for (const grant of blocked) host.revoke({ grant });
-    },
-    signOut: async ({ device }, ctx) => {
-      await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx);
-      host.revoke({ device });
-    },
-    …                                                         // shareLink, join, signOutAll
-  };
-});
-
-host.serve(Access, (_input, principal, self) => {           // remote view: owner only
-  if (!can(principal, "manage")) throw forbidden();
-  const store = self.get(AccessStore, {});
-  return {
-    variant: "owner",
-    guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)),   // names, modes; no tokens
-    removeGuest: (args, ctx) => store.removeGuest(args, ctx),
-    resetLink: (args, ctx) => store.resetLink(args, ctx),
-    signOut: (args, ctx) => store.signOut(args, ctx),
-  };
-});
-

authenticate is base code on the same host and checks AccessStore's current documents on every new connection -(it holds a reference to the store, taken with server.root at start). Live access and lasting denial stay separate, as chapter VII explained:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
EventLasting effect (data)Live effect
remove a guestguest record blockedrevoke({ grant })
reset a share linknew token; its guests' grants blockedrevoke({ grant }) for each
archive a sessiongrants of that session void while archivedrevoke({ session })
restore a sessiongrants valid again; revision bumpednone: guests reconnect
sign out a devicedevice revoked until its expiryrevoke({ device })
sign out all devicessigning key rotated, tokens clearedrevoke({ device }) for each
-

A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the -block is in Guests and authenticate refuses it. Names are unverified, cleaned, at most 40 characters, unique per -session ("Ann (2)").

-

The catalog

-

Catalog owns the session records: id, title (and whether it was edited), location, model default, pinned, -archived, timestamps. It lives on the server, persisted, and it never starts a worker: listing sessions, showing -their status or their unread marks reads server-side copies only.

-
export const Catalog = remote("p3.catalog", {
-  version: 1,
-  input: obj({}),
-  placement: () => Server(),
-  variants: {
-    owner: {
-      state: { sessions: obj({ items: arr(SessionRow) }) },
-      methods: {
-        create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs),
-        archive: method(obj({ id: str() })), restore: method(obj({ id: str() })),
-      },
-    },
-    guest: { state: { sessions: obj({ items: arr(SessionRow) }) } },     // only the guest's own session, if not archived
-  },
-});
-

Behind it sits a persisted local CatalogStore; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, -relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: the catalog does not know which -plugins exist. Archiving a session writes archived: true and revokes the session; every plugin that keeps -per-session things reconciles from catalog.sessions with a routine (chapter X), as the notes plugin did. The ports -plugin closes forwards of archived sessions that way, also after a crash.

-

On the worker: start data, location, harness

-

Three core objects exist on every worker from its start:

-
    -
  • SessionInfo (chapter IX): the start message as constant(init), with the session id, data directory and the -location the server chose at start.
    • -
  • SessionLocation { session } (working name), lifetime host: the current cwd and environment, with a location -revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it -with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core -per-conversation policy document.
    • -
  • Harness (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened -under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root -model is not configured, and resumes interrupted Durable work on start.
    • -
-

ConversationStore and the Conversation views in front of it are chapter XIX.

-

Workers: start, keep alive, stop

-

Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is -server.route(Session, { authorize, resolve }) plus a small server object that keeps per-session worker state:

-
A worker's life, as the supervisor sees it

Authorize, then start

A tab gets Notes { S1 }. authorize checks can(p, "read", "S1") and that the session exists and is not archived -(an archived session refuses, except an index rebuild that starts the worker without resuming work). resolve forks -worker/main.ts with the start data from the catalog, bounded by a 60 s readiness timeout.

-

A start failure sticks

If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and -every get fails at once with it, until the owner presses Retry. That clears the error and the related notices. -Guests see that the session is unavailable, not the error text.

-

Counting

The core facet on the worker reports host.activity ({ references, calls, jobs, lingering }) and the harness's -running Durable task count to the supervisor object, from a routine whenever either changes. The call carries -ctx.incarnation, so a report from an older worker incarnation is ignored (chapter X).

-

Idle stop

When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: -the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted -and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is -a job, so it counts.

-

Crash

If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail -(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call -resolve again with backoff; a passive viewer recovers without a click.

-

Shutdown

On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains -persisted.

-
-

Status without a worker

-

The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, -and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a -routine over the harness's state) calls SessionStatus.report(summary) on the server; SessionStatus keeps the latest -value per session and ignores calls whose ctx.incarnation is older than the newest it has seen; clients read -SessionStatus, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the -same pattern (chapter XIX).

-

Settings and notices

-

Settings (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have -one writer: a core Settings object on the server owns the file, validates writes and serializes them; every -other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches -settings, models and auth in every process and writes them from two, which is decision #21.

-

Notices are one channel: one core Notices object on the server, written by everything that fails or finishes -(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most -50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing.

-
Review: settings and file formats (#21, #22)

Single writer for settings, models and auth (#21) and versioning on-disk formats such as envs.json like documents -(#22) are open p3 decisions in design.md §26. The construction above assumes "one owner on the server, others -follow its state" for #21.

-
-
Built so far

Nothing of p3 exists. The functionality map is complete (apps/p2/docs/p3-functions/, five area files plus the -condensed README.md), and the design's app rules (design.md §25) are decided. p3 starts after the layered rebuild -and the playground's first real proof.

-
-
- -
XIX

Sessions and Conversations

The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses.

-

This chapter builds the domains a person touches most: the session list and its operations, the conversation view -and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section -names the owner, the host, and the Chord parts it uses; code appears only where it shows something new.

-

The session lifecycle

-

The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OperationOwner and formNotes
createCatalog.create, a methodabsolute directory, ~ expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used
rename, pinmethods on Catalogtitle 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically
automatic titlea routine on the search feedthe first user message, line breaks to spaces, 100 characters
agent renamean agent tool on the worker calling Catalogroot conversation only
archive, restoremethods; revocation; reconciling routinesbelow
clone, fork, relocatejobs on the catalog storeoutlive the call, cancellable, one outcome
deletea job; decision #18below
-

Archive and restore

-

Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while -keeping its storage. The catalog does none of the plugin-specific parts:

-
Archiving S1

One write

Catalog.archive({ id: "S1" }) sets archived: true in the session record, in one transaction.

-

Revoke the session

The catalog calls host.revoke({ session: "S1" }). Every guest principal of S1 has { session: "S1" } in its -revokedBy, so their objects end and their links close on the server and on the worker (chapter VII). New -connections are refused by authenticate, because grants of an archived session are void.

-

Stop the worker

The supervisor reconciles from catalog.sessions with a routine: an archived session's worker is stopped, and -authorize refuses to start it again.

-

Plugins reconcile

The ports plugin's routine on catalog.sessions closes forwards of archived sessions. The search plugin marks the -session's rows archived. If the server crashed between the write and these steps, the routines run again with the -current value at the next start: nothing is a notification that can be lost.

-

Restore

Catalog.restore clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards -stay closed.

-
-

Clone, fork, relocate

-

Each is work that outlives its call, can be cancelled and has one outcome, so each is a job (chapter XI) on the -catalog's local store, keyed by the owner's request id. They differ in what they pause:

-
    -
  • Clone copies the session's store, uploads, downloads and artifacts under a new id, with the source worker -paused (refused while work runs) and title "(copy)"; directories 0700, files 0600, no links, no overwrite; a -partial copy is removed on failure.
    • -
  • Fork before a user message copies the active context as of the entry before it into a new session (title -"(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, -the new one is seeded empty. A failure archives the copy.
    • -
  • Relocate (cwd, environment or both): refused while the session is busy (naming the running subagents) or when -a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates -the record, and moves the running worker in place by updating SessionLocation: natives that follow the location -switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable -commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence.
    • -
-
Review: jobs as objects (#9)

The design defers whether a long relocation or a login is a record inside its owner (watched through running) or an -object of its own (Relocation { id }). The construction above uses records. Decide when login is ported -(design.md §15, §26).

-
-

Delete

-

p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session -(store.retireByTag(SessionTag, id), chapter V), the worker's databases (the Durable store), the session's files, the -search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to -this outcome (design.md §25); how plugins take part is open.

-
Review: session delete (#18)

Options in design.md §26: per-session objects plus SessionTag plus one session directory (delete is retire by tag -and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: -whether the search index is per session (deleted with it) or global with a hook. Who holds the store for -retireByTag is not specified either.

-
-

Status, children, unread: reports to the server

-

The session list must show status, subagents and unread marks for sessions whose workers are not running, and must -not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the -current summary, the server object keeps the latest value per session (persisted where it must survive a restart), -ignores calls from an older ctx.incarnation, and clients read the server object.

- - - - - - - - - - - - - - - - - - - - - - - -
SummarySource on the workerKept on the server
status: running, error, task count, busy conversations, usage and costthe harness's task graph and usageSessionStatus, in memory: a stopped worker is "not running"
children: name, model, thinking, tools, cwd, last entry, archivedthe subagents plugin's child listChildren, persisted
latest root answerthe search feedUnread: compared with the owner's read mark, persisted, shared by all owner devices
-

Guests see their own session's status without the error text. The owner's read mark is written when the root -conversation is visible, and a new session starts as read.

-

Conversations

-

The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older -pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them:

-
    -
  • ConversationStore { conversation } (local, on the worker): opens the Durable watch as its native, reaching -the harness through the core-private HarnessRuntime registry entry, and exposes a window as a derive: the -newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays -put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list -(chapter VIII's window pattern).
    • -
  • Conversation { session, conversation } (remote, placement Session): the per-principal view. Variants by -can(): owner (everything, including model, thinking and handoff), writer (guest write: send, steer, stop, -compact, shell), reader (the window only).
    • -
-
host.serve(Conversation, ({ session, conversation }, principal, self) => {
-  const store = self.get(ConversationStore, { conversation });
-  const policy = self.get(ConversationPolicy, { conversation });         // core: readOnly, followsSessionCwd
-  const author = authorOf(principal);                                    // structured: kind, id, display name
-  if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page };
-  return {
-    variant: can(principal, "manage") ? "owner" : "writer",
-    view: store.window,
-    page: store.page,                                                    // older history: returns data
-    send: ({ text, attachments, mode, requestId }, ctx) => {
-      if (policy.value.readOnly) throw forbidden("read-only conversation");
-      return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx);
-    },
-    stop: (_args, ctx) => store.stop(author, ctx),                       // allowed even when read-only
-    …
-  };
-});
-

What each function becomes:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FunctionHow
send, retried sendDurable submit with the author's requestId: a retry with the same id is admitted once (Durable dedupes, chapter XIV)
steer or queue while busywhenBusy: "steer" or a follow-up; the inbox is docs["pi.inbox"] in the window
attributionthe view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text
wait for prompt resourcesthe first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows)
stop, dequeuewithdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their restored drafts
restored draftsa keyed document per (conversation, author), tagged SessionTag, persisted in the store; claimed once, first tab wins
read-only conversationsthe core per-conversation policy document, written at createConversation({ init }) by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed
history pagesa method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included
model and thinkingowner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default
compaction, handoffDurable operations through the store
-

Drafts in the composer are UI state, not Chord state: the web shell keeps one draft in sessionStorage, the TUI -keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged -("Sending when connected"). Only restored drafts, which the worker produces, are documents.

-

Shell commands

-

!cmd runs a command in the conversation's cwd on its environment and records the result as an entry; !!cmd does -the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is -at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the -run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is:

-
const ShellJob = defineJob("chat.shell", {
-  version: 1,
-  args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }),
-  progress: obj({ output: str() }),
-  result: ShellResult,
-  concurrency: 1,                                                   // one per conversation
-});
-
-const results = self.state(ShellResults);                           // keyed by run id: dedupe after completion
-const shell = self.job(ShellJob, {
-  run: async ({ command, include, author }, ctx) => {
-    const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" });
-    const env = await environmentFor(ctx);                          // the conversation's environment, chapter XX
-    const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) },
-      (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); });
-    const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted };
-    await conversation.submit(
-      { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx);
-    return data;
-  },
-  finish: async (done, tx) => {
-    (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error };
-  },
-});
-

The request id shell:${ctx.jobId} makes the entry idempotent: an attempt interrupted after the write cannot write -it twice. Cancel is shell.abort(id), allowed for the command's author and the owner. A retried runShell with the -same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after -completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does -not depend on viewers.

-

Chat mode

-

Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new -subagents start with it off. Because it must fork and rewind with the conversation, it is transcript-bound: a -Durable conversation document, not a Chord document. Messages without @pi become display-only Durable write -entries (author, text, attachments, reply quote); a message with @pi becomes an input with a digest of the chat -since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply -quote is derived on the worker from the entry id the client names: the client never supplies author or quote text.

-

Subagents and one-shot agents

-

Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one -subagent tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and -each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a -restart sends nothing twice. Chord adds only three things:

-
    -
  • the child list for the sidebar is reported to the server (Children, above), so it shows without a worker;
    • -
  • a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in -the core per-conversation policy document when it is created, so chat and relocation enforce it without knowing -the subagents plugin (design.md §25 porting guideline);
    • -
  • archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots.
    • -
-

Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with -the session's model choice.

-

Messages between sessions

-

The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and -calls a server object through chord(fn, context) (chapter XIV):

-
const peerSend = defineTool({
-  name: "session_send",
-  execute: (args, api, context) => chord(async (ctx) => {
-    const peers = await ctx.get(Peers, {});                       // server, worker variant
-    await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx);
-    return { output: [{ type: "text", text: "sent" }] };
-  }, context),
-});
-

Peers checks that the sender is a root conversation of a non-archived session, refuses sending to itself and -archived recipients, then gets the recipient's Conversation (which starts its worker) and submits with the -sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried -tool call delivers once. The sender's -incarnation is ctx.incarnation; a call from a replaced worker is refused. Only owner-initiated turns may use the -tool; guests never.

- -

The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no -Durable internal inputs) for title, cwd and message search across sessions, active or archived.

-
WORKER S1SERVERDurable commitin any conversationwakea routine on the harness's commitsignalread from cursor200 entries per pageSearchIndexnative: SQLite FTScursor per conversationmoves after the whole rangequeryterms, scope, at most 41 hitsreport(page)Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing.
Figure 13. The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query.
-

The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash -the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting -an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, -snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" -forward and supplies the latest root answer for unread marks.

-

Presence

-

Viewers per session are host.references(Conversation) on the worker, counted per client (chapter VIII), and -reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence -is not keep-alive: the supervisor uses host.activity, which also counts calls and jobs.

-
Built so far

None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, -child conversations, background tasks) and the Chord layers L3 to L9 (not built).

-
-
- -
XX

Machines, Files, Models

Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3.

-

The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, -terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are -places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, -three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per -thing.

-

Environments

-

The envs plugin owns the environment configuration: name, host, user, port, identity file, shell and its -variables, plus a managed ed25519 key per environment and its own known_hosts (kept apart from the user's, aliased -per environment name).

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FunctionOwner and form
list, add, edit, delete (refused while a session, archived too, uses it)Envs on the server: a persisted document behind an owner-only view
validation (no leading -, no whitespace or control characters in host and user)the view's methods, before the write
a hand-edited envs.json is picked up; an invalid file keeps the last valid set and shows an errora native file watcher (poll fallback) in the envs store; one writer (#21)
environment revision: a hash of the config, part of its identitya derive
connection test (connect, deploy the daemon, platform, latency, warnings)a job; the result per revision in a document
unknown or changed host key: fingerprints, trust after confirmation, forgeta flow (chapter XII) on the server
browse directories (~, relative paths, at most 10k scanned, 1,000 shown)a method with a call-scoped SSH session (chapter IX) or the pooled connection
a changed environment restarts its sessions' workers once idlea routine on the envs store and the status reports
-

Execution in the agent. The worker's harness asks HarnessEnvironments for an environment per target (chapter -XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over -SSH through the pi-env daemon, deployed per platform and connecting at the first operation. A remote session never -falls back to local tools. The environment follows SessionLocation: after a move, the old connection closes once -the move has committed and watchers reopen at the new location.

-
// envs/worker.ts
-export default defineFacet((host, self) => {
-  const location = self.get(SessionLocation, { session });
-  const pool = self.get(EnvPool, {});                              // one owner of SSH connections (#20)
-  self.contribute(HarnessEnvironments, (target) =>
-    target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd));
-});
-
Review: connection pools (#20)

SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, -terminal host). design.md §26 asks for one owner with a pool or deliberate per-use connections. EnvPool above is -the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or -key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change -rarely.

-
-

Workspace and file index

-

The workspace plugin serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: -list a folder (at most 2,000 entries, .git hidden), read a file (1 MiB, binary detected), git status, diff, log and -single commits, literal text search, and all paths for name filtering. Every query is a method returning data, not -state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, -bumped by a native watcher of the tree, HEAD, the index and config (debounced 100 ms); viewers reload what they show -when it changes.

-

Two rules live in one place each instead of three:

-
    -
  • Confinement: path rules, a canonical root, no .git, the app's private data directory denied (also excluded -from search). One helper used by listing, reading, uploads and the file index.
    • -
  • Git hardening: no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s -timeout, output caps.
    • -
-

Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that -reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp -file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it.

-

The file index for @ completion belongs to the same owner, so completion consumes the index instead of scanning -on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per -second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until -sent; sending them is an ordinary Conversation.send.

-

Terminals

-

Terminals must survive server and worker restarts, so their PTYs live in a detached terminal host process, one per -data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals -plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the -session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). -Terminals are owner only.

-

Bytes do not go over object links. Attaching opens a channel (chapter XIII) with a single-use token (30 s): JSON -control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a -viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current -screen from a headless emulator with 10k lines of scrollback.

-
Review: the terminal host process (#23)

The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals -end), or deliberately outside it with a "host runs older code" notice as in p2? Open in design.md §26.

-
-

Session files

-

Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's send_file tool, versioned -artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all -per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory -layouts for them. The proposal is one owner:

-
export const SessionFiles = remote("p3.session-files", {
-  version: 1,
-  input: obj({ session: str() }),
-  placement: () => Server(),                                  // readable without a worker
-  variants: { writer: { … }, reader: { … }, worker: { … } },
-});
-

SessionFiles { session } on the server holds the index (per kind, readable without a worker), the chunked upload -and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied -into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a -sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types.

-
Review: session files (#17)

Proposed in design.md §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and -512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote -environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents.

-
-

Ports and Radius

-

The radius plugin keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in -by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, -60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two -kinds of channels:

-
// radius/server.ts (the entry point: one line, as in chapter I)
-radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } }));
-radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value));
-

Object links become owner principals with access: "relay": owner, but treated as remote (no localhost-only -powers). p2 had to fake a loopback request with a secret header for this; with accept that is gone.

-

Forward channels go to localhost:<port> (Host, Origin and Location rewritten; 502 when nothing listens; anonymous -visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, -radius.forwards, and the ports plugin contributes its forwards (at most 50, random ids, a public toggle that -needs confirmation, an agent tool ports for top-level owner-initiated conversations). Without the ports plugin the -table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The -ports plugin closes forwards of archived sessions with a routine on catalog.sessions (chapter XIX).

-
Review: may a plugin call accept?

design.md §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If -radius is a plugin, its facet needs the host's accept, which no other part of this guide gives a plugin. The -alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified.

-
-

Models and credentials

-

The models plugin owns the model runtime (pi-ai's models with custom providers and the model cache) in a -host-lifetime object, and the credentials. It contributes the runtime to HarnessModels (chapter XIV), so after a -models reload the next request uses the new runtime while a running stream finishes on the old one. Credentials -never go to clients: the views expose providers, login methods and which type is signed in, never tokens. Every -secret is registered for redaction in one place, the log capture.

-

Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), -logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are -settings (chapter XVIII).

-

Login

-

A login is the guide's flow example (chapter XII): a job on the server that runs pi-ai's login, and a flow that -shows its steps to the starting owner.

-
const LoginJob = defineJob("models.login", {
-  version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }),
-  concurrency: 1,                                                    // one login at a time on the server
-});
-
-const login = self.job(LoginJob, {
-  run: async ({ provider, type }, ctx) => {
-    await runtime.value.login(provider, type, {
-      signal: ctx.signal,                                            // 10 minute deadline, cancel, leaving
-      prompt: (prompt) => steps.ask(prompt, ctx),                    // text, secret, select, manual_code
-      notify: (event) => steps.notice(event, ctx),                   // info, auth_url, device_code, progress
-    });
-    return { ok: true };
-  },
-  finish: async (done, tx) => { /* record the outcome; refresh the model list */ },
-});
-

prompt is a promise-based callback inside the work; the resolver stays in the job (steps.ask records the step in -the flow state and waits for answer), and nothing crosses an object boundary. On a relayed connection the device -code method is selected automatically. The flow is visible only to the owner principal that started it; others are -refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine -aborts the job. Codes, URLs and typed secrets are redacted from logs.

-

Prompt resources

-

AGENTS.md, skills and templates are read in the session's environment, watched (poll fallback, last valid state), -and expanded on the worker before sending. The prompt plugin owns them per session on the worker, contributes the -system prompt section and skills to HarnessExtensions as readable contributions (a change recomposes), and exposes a -readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers -reopen at the new location.

-

Code mode

-

Code mode is a plugin (chapter XIV): its pi-codemode sandbox is a native; scripts call tools as nested Durable calls -and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the -old sandbox for at most 30 s. The on/off switch is a setting.

-

Logs and notices

-

Log capture tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, -redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB -each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. -Notices are the one channel from chapter XVIII; dismissal stores only notice ids.

-

Dev and reload

-

The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; -the reload_plugins agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for -base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a -successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; -the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (0700) accepts development -commands without a network admin port.

-

The shells

-

The web and TUI shells are UI facets over the objects above. Two rules matter for Chord:

-
    -
  • UI state survives navigation, reload and reconnect on purpose. Composer drafts, pending sends with their request -ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect -re-holds and gets snapshots; nothing is lost that was not in flight.
    • -
  • Leaving is never derived from an unmount. A plugin reload remounts components (chapter XV). Cancelling a login, -closing an overlay or giving up a lease is an explicit action or a dropped reference.
    • -
-

The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or -credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a -confirmation. That is app code around Chord, not part of it.

-

Transport

-

The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded -on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with -snapshots after a reconnect, and a protocol version that makes an outdated web page reload.

-
Built so far

None of p3 exists. pi-ai's login API, pi-codemode and the pi-env daemon exist and are used by p2 today; the HTTP -component and channels are deferred until after the kernel's first real proof (chapter XIII).

-
-
- -
XXI

Other Ways to Do It

Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs.

-

Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is -Cordis, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework -since 2019, and its most visible user, DeepSeek Harness (dsh), DeepSeek's open-source agent harness, whose -motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: -who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model -reaches beyond one process.

-
Sources and status

DeepSeek Harness at commit 687f282 (release 0.2.1-rc.2), which vendors Cordis as @deepseek-ai/cordis 4.0.5 (Cordis -RC7 with local changes, vendor/README.md): its Cordis tutorial (docs/cordis-tutorial/), docs/architecture.md, -docs/subsystems/persistence.md, docs/subsystems/core.md and packages/boot/hmr/README.md; the Cordis paper "A -Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 -is the design this guide describes. The comparison is between a working system and a design.

-
-

Cordis in one example

-

A Cordis plugin is a function (or an object with apply, or a Service subclass) that receives a context. It may -declare the services it needs with inject, and everything it does to the world goes through the context:

-
export const name = "greeter";
-export const inject = ["tools"];                       // activate only while a `tools` service exists
-
-export function apply(ctx: Context) {
-  ctx.tools.register(defineTool({ name: "greet", /* … */ }));   // a registration is an effect: undone on unload
-  ctx.on("session/start", (s) => log(s.id));                     // an effect-bound listener
-  ctx.effect(() => {                                              // any other side effect, with its disposer
-    const timer = setInterval(work, 200);
-    return () => clearInterval(timer);
-  });
-}
-

Each plugin instance is a fiber with states PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED (or FAILED). -Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs -the disposers. The paper calls this temporal composability: a component can be removed at any time and reverses its -own effects. Its counterpart, spatial composability, is the dependency side: inject is live, so a plugin starts -when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The -paper's claim is that independently written components can therefore be combined and replaced without disturbing each -other. The next sections check how far that holds.

-

Services are named capabilities on the context (ctx.provide("x", value) or class X extends Service). Names are -runtime strings in one flat namespace per application; TypeScript declaration merging makes ctx.x typecheck, but -nothing checks the service's shape at runtime, and optional dependencies are probed with ctx.get("x"). Scopes come -from the context tree: ctx.extend() derives a child context, ctx.isolate(name) gives one subtree its own binding of -a service, so two parts of an app can use two different shell providers.

-

Hot reload follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and -loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin -that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and -mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount.

-

DeepSeek Harness on Cordis

-

dsh applies "everything is a plugin" to an agent: the model adapters (ctx.llm), the tool registry (ctx.tools), the -session log (ctx.sessions), the agents (ctx.agents) and the agent loop itself (ctx.agentLoop) are plugins and -services. A running dsh is a plugin tree composed at boot from a profile (web, headless, sdk, acp) that -stacks bundles of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent -loop is a config patch.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ConcernHow dsh does it
sessionsa durable, append-only event log per session (JSONL backend, single writer, flush as the durability barrier); model history is derived from the events
crash mid-turnthe log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic turn/end { reason: "interrupted" }
toolsctx.tools.register(defineTool(…)), an effect of the registering plugin; calls go through pre/execute/post policy waterfalls
approvalclosed by default; allowed-once is the only grant; plugin installation needs full access or per-call approval
reloada source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs
processesone plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams
distributiona Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes
subagentslocal subagents with their own durable sessions, and external ones
-

What the guarantees are in practice

-

Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's -framing. An earlier review for p2 came to the same result (apps/p2/docs/p3-reviews/dsh-compare.md).

-
    -
  • Reversal is cooperative and unordered. _unload runs all of a fiber's top-level disposers concurrently -(Promise.all in vendor/cordis/src/fiber.ts); order holds only inside one effect, and the primer tells authors to -"keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A -disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose -cancel throws and warning that "the work may be orphaned" (packages/jobs/jobs-local/README.md). Reversal is as -good as every disposer, and nothing checks that a plugin registered all its effects.
    • -
  • Behaviour depends on registration order, and a reload changes it. emit, waterfall, serial and bail -listeners run in registration order (docs/cordis-primer.md). Registering appends the listener (push in -vendor/cordis/src/events.ts, unless prepend), so when a plugin reloads, its listeners move to the end of the -list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits -now sees a different decision first.
    • -
  • Dependencies are strings, and every provider reload restarts its dependents. A fiber's activation key is the -concatenation of its injected providers' fiber ids (_refresh in fiber.ts), so replacing a provider unloads and -reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every -consumer" (AGENTS.md): there is no versioned contract between a provider and its consumers.
    • -
  • Reloading a core service ends live work. Unloading the agent provider "stops and drains every live handle it -made" (packages/core/agent/README.md), so running agents stop; reloading the session persistence backend under -live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" -(.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md).
    • -
  • Registrations leave a gap. A reloading plugin's tools and other registry entries are removed with its old fiber -and added again by the new one; anything that looks them up in between does not find them.
    • -
-

None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components -interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register -every effect, order nothing by registration, and hold no state that must outlive a reload.

-

Side by side

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Cordis and dshChord v2
unita plugin instance (fiber) in one context treean object per (type, input, principal), placed on a host (chapters II, III)
side effectsrecorded on the fiber with ctx.effect and friends; on unload the disposers run concurrently, each as good as its author made itdeclared through self (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI)
dependenciesservice names as strings in one flat namespace, via inject; a provider reload restarts every dependenttyped contracts via get(Type, input); a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII)
orderinglisteners run in registration order, which changes when a plugin reloadsregistries compose with a declared compose function; nothing depends on load order (chapter XIII)
where code runsone process; remote UIs over RPC to the Hosta tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII)
who is callingnot a framework conceptprincipals stamped by links, variants per caller, revocation (chapters III, VII)
statethe plugin's own business; dsh's session log is a servicedocuments written in transactions, persisted, versioned and migrated (chapter V)
reloadunload the old instance, load the new one: in-memory state and running work of that plugin endpure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV)
a running agent turn during reloadentries of changed plugins stop; unloading the agent provider stops its live agentsthe harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV)
crash recovery of a turnthe turn is closed as interruptedpi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV)
contracts and versionsTypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schemacontract versions with fingerprints and provider-side mappings for old consumers (chapter XVI)
statusin production in Koishi since 2019; dsh releaseda design with the contract layer built
-

What the comparison says

-

The core idea is the same. "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are -two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. -Both reject the module-level timer and the stored callback for the same reason.

-

The difference is what survives a reload. Cordis reloads at plugin granularity: the old instance unwinds and the -new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory -state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on -registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as -records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what -holds OS handles. That is more machinery: transactions, a store, versions on every document and contract.

-

The other difference is reach. Cordis composes plugins inside one process, and dsh adds RPC to its own clients. -Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are -principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per -session and guests in other browsers, needs that; a single-process agent does not.

-
Review: what a reload does to a running tool call in dsh

The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live -agents (packages/boot/hmr/README.md, docs/subsystems/core.md). They do not say what happens to a tool call in -flight when only the tool's plugin reloads; the comparison above does not claim either way.

-
-
- -
∴

What's Next

Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other.

-

The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, -factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and -mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the -preface while the agent keeps running. This last chapter says how much of that exists and what is still open.

-

Status

-
~10klines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only
1 of 11layers of the rebuild started: L1 contracts, built and reviewed
6–7ktarget size of Chord v2 without tests
-
    -
  • The design is decided except the points marked below. packages/chord/docs/next/design.md is the target; -packages/chord/docs/next/decisions.md is the log and wins where they differ.
    • -
  • The rebuild goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and -garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and -derive; L3 documents and store; L4 objects; L5 references and ctx; L6 protocol and links; L7 relay, routes, -revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord.
    • -
  • Built: the contract layer (src/next/contracts/, commit 7d2bb8ea8, chapter XVI). In pi: Delta, -pi-telemetry, chord/context, chord/telemetry-sqlite, pi-ai and Durable telemetry, Durable's required -checkpointWhen.
    • -
  • Next, in order (apps/p2/docs/p3-handoff.md): decide Q12 and Q15, which the L1 spec needs; write the spec for -the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in -packages/chord/docs/next/layer-1.md); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect.
    • -
  • In pi: mapShared and bytes as atomic leaves in Delta's diffRevisions.
    • -
  • Then the playground web shell (React, esbuild) with the reload control and a preview at p3.mariozechner.at, -and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and -channel components, then p3, domain by domain.
    • -
-

Deferred decisions

-

Kept with stable numbers in design.md §26, each waiting for a concrete case:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#TopicWhenChapter
9jobs as objects: a record inside the owner, or an object addressed by its idwhen login is portedXI, XIX
11where a flow lives: a keyed document of its owner, or an object LoginFlow { id }when login is portedXII
14project plugins: per-directory plugins with trust prompts and snapshotsown pass over plugin-loading.mdXV
16a Durable Object profilewhen a DO target becomes concrete
17session files: one SessionFiles { session } owner on the server (proposed)p3 designXX
18session delete: retire by tag plus a session directory, or owner hooksp3 designXIX
20connection pools: one owner of SSH connections per environment, or per-use connectionsp3 designXX
21config files with a single writer: one owner on the server, others followp3 designXVIII
22file formats (envs.json, downloads, uploads) as versioned interfacesp3 designXX
23the terminal host process: inside the revision or deliberately outsidep3 designXX
-

Open from review round 3

-

Decisions (apps/p2/docs/p3-reviews/v2-review3-summary.md; Q1 to Q10 are decided):

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#ProblemProposal
Q11a singleton that does not exist yet: undefined or initial()?singletons read initial(); undefined only for absent keyed documents
Q12stripping arguments (R7) against "nothing is copied"copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft)
Q13ctx.incarnation on relayed callsrelays stamp the origin host's incarnation, as they stamp the principal
Q14a writable holding functions exposed as a state memberonly members with a JSON schema in the contract are exposable, checked at serve
Q15arguments of local methods: validated or notapply R7 to local too, or keep local as the one exception
-

Cuts the owner may choose (each removes a primitive this guide teaches):

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
#CutWhat it costs
C3shrink jobs to a runner over documents (no versions, accepts, blocked, persist), or move jobs out of the kerneljob versioning, or self.job for core-only users
C4routines: on and loop onlyevery's due time across reloads
C5merge lifetime: "host" objects into the faceta plugin's always-on natives share one failure domain
C6remove broken() for opens and retryOnnothing; reload and Retry still work
C7remove provider-side lingerten lines of app code
C8remove state: false (method-only references)a method-only consumer pays one snapshot
C9remove previous from derivesderives that used it move to mapShared
C10one cell(initial) for constant and writablenothing
C11remove ctx.trace (use traceIn(ctx, …))nothing
C12remove call credits (the byte window covers calls)nothing
C13two timeouts (closeTimeout, readyTimeout) instead of eightper-thing overrides
C14remove the handle.change and handle.retire shorthandsnothing
C15remove inVariant and expectVariant (keep RefIn)nothing
C16remove facet-owned mutable documents and their transaction exceptionatomic writes across an object and its facet move to one owner
C17no harness-specific start-barrier exceptionfactories before the barrier cannot depend on the harness
-

Where the sources disagree

-

This guide follows the decision log where it is newer than the design, and the design where it is newer than -everything else. The places that need a fix in the sources:

-
Review: drift in the written sources
    -
  • design.md §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to -C17.
    • -
  • design.md §3 has a garbled sentence, "Values are method arguments are stripped".
    • -
  • apps/p2/docs/p3-handoff.md §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and -R7 to R26 are open, and "resume R6 next".
    • -
  • kernel.md and step-*.md describe the frozen code: host values, a development freeze, consumer-side linger, -JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target.
    • -
  • chord-next-walkthrough.md lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed -by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome -{ ok } (now { kind }), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), -and hooks using host.root (now chord(fn, context), Q6).
    • -
  • apps/p2/docs/questions.md uses self.resource and replace (both rejected) and host.root in hooks.
    • -
  • apps/p2/docs/p3-reviews/v2-cost-model.md: the measurements are current, its "Decisions this needs" section is -not (chapter XVII).
    • -
  • L1's Implementation<C> shape differs from the factory return value used in the design (chapter XVI).
    • -
-
-
- -
A

API Reference

Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built.

-

Hosts

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CallMeaningChapter
createHost<P, R>({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })a host; with an address it is addressable and its own code acts as principal; without, a clientI
host.load([{ name, facet }])load facets by plugin name; base code passes its core facet the same wayI
host.connect(link, { principal })the link to the parent (at most one)I
host.accept(link, { principal })a link from a client; every request on it acts as principalI
host.route(Kind, { authorize, resolve })how to reach hosts of a kind; resolve returns { link, principal }I
host.revoke(r)end matching objects and links here and belowVII
host.root(fn, { principal?, signal?, telemetry? })a call scope with a ctx for code outside objectsVI
host.get(Type, input, { signal, state?, parkTimeout? }), host.hold(…)a reference from outside objects; release() ends itVI
host.serve(Type, factory, { lifetime?, linger? })serve an object type (in a facet)II
host.registry(Registry)the composed value of a registry, a readableXIII
host.references(Type)who holds references to objects of a type on this host (presence)VIII
host.activity{ references, calls, jobs, lingering }, for keep-aliveI
host.close()end the host: objects, natives, linksI
-

Inside a factory: self

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MemberMeaningChapter
self.state(Definition)a document handleV
self.tx(async (tx) => …, ctx)a transaction on the object's lineV
self.derive(name, [sources], fn)a derived readableVIII
self.get(Type, input)a reference held for the object's lifeVI
self.open(open, close, { closeTimeout? })a native; .value once openIX
self.routines.on(readable, fn), .every(interval, fn), .loop(fn), .custom(…)standing behaviourX
self.job(Job, { run, finish? })owned work that outlives its callXI
self.contribute(Registry, value)a contribution (facets and lifetime: "host" objects only)XIII
self.writable(initial)a host-local readable of any in-process value, with setXIII
self.retryOn(readable)a trigger that retries a broken activationVI
-

Document handles: value, get(key) (keyed), keys, change(fn, ctx) / change(key, fn, ctx) and retire(ctx) / retire(key, ctx) (shorthands for -one-document transactions). Transactions: tx.draft(handle, key?, { tags? }), tx.retire(handle, key?), -tx.start(job, args, { key? }). Jobs: job.start(args, ctx, { key? }), job.abort(id), job.running, -run.outcome(ctx).

-

References and ctx

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
MemberMeaning
ref.<member>a state member (readable: .value, subscribe) or a method
ref.state, ref.onState(fn)loading, live, failed (cause, retryAt), released
ref.retry()retry a failed reference now
ref.release()end a reference taken outside objects
ref.membersthe variant's members
ctx.signal, ctx.principal, ctx.incarnation, ctx.getcancellation, caller, calling host's incarnation, call-scoped references
ctx.telemetry, ctx.trace(name, attributes, fn)the current span; a child span with a child ctx
ctx.jobId, ctx.progress(value)in a job's run
-

Definitions

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameWhat it declaresChapter
hostKind(name, input?)a kind of host and its addressI
remote(name, { version, input, placement, variants })an object type reachable across linksII
local(name, { version, input, members })an object type on its own hostIV
method(args?, result?)a method in a base declarationII
evolve(prev, (e) => …), checkAccepts(contract, accepts)the next contract version; the provider's mappings (built)XVI
Implementation<typeof C>the type of a provider for the current version (built)XVI
str num int bool nul lit oneOf union obj arr map opt bytes unknown, Staticschema helpers (built)II
defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })a documentV
defineTag<T>(name)a tag; store.retireByTag(tag, value)V
defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })a job typeXI
defineRegistry(name, { version, compose, accepts? })an extension pointXIII
bindFacet<P>()base code's defineFacet, with factories' principals typed as the app's P (working name)III
defineFacet((host, self) => …)a plugin's setup for one kind of host; imported from base code (@studio/core)II, III
constant(value)a readable that never changes (start data)IX
broken(error)a failure that is not retried by the backoffVI
inVariant(ref, …), expectVariant(ref, …), RefIn<…>narrowing a reference by variantIII
mapShared(fn)an array map that keeps unchanged items identical (planned in Delta)VIII
OutputBuffer, Progressbounded output and throttled progress (@earendil-works/chord/output)XI
memoryLinkPair(), ipcLink(child), webSocketLink(ws)link adaptersI
DocumentStorage, registerDocumentStorageConformancethe storage port and its shared test suiteV
-
- -
B

Timeouts and Defaults

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Timeout or defaultBoundsDefaultOverride
closeTimeouta native's close, then reported as a possible leak30 sper native
call drainan ending object's own call handlers= closeTimeoutper type
readyTimeoutactivation; an awaited get60 sper get
park deadlineparked calls (link drop, loading, rollout)60 sper get (parkTimeout); per type
routine handovera new routine waiting for the old handler on reload10 sper routine
finish retrybackoff of a throwing job finish1 s to 60 sper job
rollout installone host installing a revision60 snone
lingeran object kept alive after its last referenceper typeper type
backoffretries of transient failures, per address1 s doubling to 60 snone
evictionkeyed persisted documents idle in memory10 minutes
client WebSocket limitsper part16 MiB compressed, 64 MiB decodedper link kind
compressionJSON parts compressed at their originabove about 16 KiB
worker keep-alive (p2, app code)idle time before the supervisor stops a worker5 minutesapp
-
- -
C

Versions

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SurfaceVersionOn changeChapter
object typeversion plus fingerprintevolve and checkAccepts; otherwise contract_mismatchXVI
documentversion plus schema fingerprintpure migrate on access; no bump fails brokenV
jobversionaccepts on outcomes and running; blocked withoutXI
registryversionaccepts on contributions; unmatched ones left out and reportedXIII
nativenonerestarts on reloadIX
Durable tasks and documentsDurable's versionDurable's migrateXIV
link protocolprotocol versionrefused; hosts update togetherXVII
UI components, Chord and app APIsnone yetexposed as they are
-
- -
D

Errors and States

-

ChordError has a stable code and a finer reason; on the wire it is { code, reason?, message, details? } -(specified in layer-1.md, not built).

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CodeMeaning
invalidinput, arguments or placement rejected (invalid/input, invalid/args, invalid/placement)
contract_mismatchthe consumer's version and fingerprint are neither current nor accepted
forbiddenthe caller may not do this
gonethe object or hold ended
not_readynot activated in time
releasedthe reference was released
too_largea part exceeded its size limit
unavailablecould not be reached before the deadline (unavailable/locked and others)
abortedcancelled by a signal
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
TypeValues
FailureCause{ kind: "transient"; error } (retried with backoff) or { kind: "broken"; error } (retried only by a trigger)
reference stateloading, live, failed (cause, retryAt), released
job outcome{ kind: "done"; result } or { kind: "failed"; error: "threw" | "aborted" | "lost" }
job phaserunning, finishing, blocked
rollout status of a providerupdating (calls park), rejected (fail at once), same revision (contract_mismatch)
host install reportinstalled, installed_failed, rejected
-
- -
E

Rejected Designs

-

From design.md §31, grouped. Each was considered and is not coming back without a new reason.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
AreaRejected
reloaddrain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; restart_required; host values and a development freeze
stateversion fences without transactions; volatile documents; per-group lines; optimistic transactions; tx.get; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store
objects and referencesresources replaced in place (replace, deps, stale handles); revoking proxies on .value; consumer-side linger; detached references counting toward presence; an optional flag on references; opaque members; callbacks as method arguments; generic call or request ids; a stream type in Chord
registriesregistries created at runtime; entries resolved through mutable indirection; generic registry keys with a conflict flag; numeric middleware priorities; registry composition as a staging check
principals and placementprincipal field patterns and re-stamping for revocation; storeKey; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete
wirean outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app
process and toolinga topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; defaultProjectTrust: "always"; ceremony (native/ directories, defineNative, lints); weakening type checking
-
- -
F

Glossary

-

Adapted binding: a consumer bound at an older accepted contract version, served through the provider's mappings. -Base code: the app's own code, imported statically; a change restarts the hosts. Client: a tab or TUI host, -not addressable. Contract: an object type's declaration (name, input, placement, members, version). -Contribution: a value a plugin adds to a registry. ctx: what every call receives. Derive: a value -computed from readables. Document: a named JSON value an object owns. Facet: a plugin's code for one kind of -host. Factory: the synchronous function that declares an object. Fingerprint: a hash of a contract's or -schema's normal form. Flow: input into running work, kept as state by its owner. Host: one participant of the -app, with an identity and an incarnation. Incarnation: the id a host gets on every start. Input: an object's -key among objects of its type; with the type and the caller's principal key it identifies the object. Job: owned work that outlives its call. Link: an ordered connection between -two hosts. Local type: an object type reachable only on its own host, without principals. Native: an OS-level -value opened with self.open. Park: a call waiting for a reference to become live again. Placement: the -function from input to host address. Plugin: a directory with a package.json, reloadable. Principal: who -calls, stamped by the link. Principal key: the string that identifies a principal in object identities. Pure -object: an object without natives; swapped in place on reload. Readable: a value plus its ordered changes. -Reference: what get returns. Registry: an owner's extension point. Relay: a host forwarding messages -between two links, rewriting only headers. Reload: swapping a plugin's code in running hosts. Restart: ending a -process. Revision: the complete code set the hosts should run. Revocation: ending live access for principals -matched by revokedBy. Routine: standing behaviour of an object. Server: the addressable root host. Shared -store: a local object that holds data once behind per-principal views. Tag: a typed label on documents, used to -retire them together. Transaction: an atomic set of document writes on the object's line. Variant: the set of -members a principal gets. View: a per-principal remote object in front of a shared store. Worker: a host the -server starts for one session.

-
- -
G

Sources

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
SourceUsed forWeight
packages/chord/docs/next/decisions.mdevery decision, review rounds 2 and 3wins where newer
packages/chord/docs/next/design.mdthe target design, all chaptersauthoritative otherwise
packages/chord/src/next/contracts/ and its README.mdchapter XVIbuilt code
packages/chord/docs/next/layer-1.mderrors, inputs, argument checksdraft spec
apps/p2/docs/p3-reviews/v2-review3-summary.mdopen Q11 to Q15, cuts C3 to C17current
apps/p2/docs/p3-reviews/v2-cost-model.mdmeasurements, per-hop costscurrent except its last section
apps/p2/docs/p3-functions/chapters XVIII to XXcurrent
apps/p2/docs/p3-handoff.mdstatus and order of work§3 partly stale
packages/durable/README.mdthe harness, submit, tasks, subagentscurrent
chord-next-walkthrough.md, kernel.md, step-*.md, apps/p2/docs/questions.mdbackground onlylag the decisions (see What's Next)
DeepSeek Harness at 687f282 (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512)"Other Ways to Do It"external, built systems
-
- -

Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README

-
- - diff --git a/packages/chord/docs/next/explainer/src/00-front.md b/packages/chord/docs/next/explainer/src/00-front.md deleted file mode 100644 index 36d62e8d0..000000000 --- a/packages/chord/docs/next/explainer/src/00-front.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -title: Chord v2: a guide to building apps that reload -cover: Chord v2 / A Guide -kicker: A guide · Chord v2 target design · 2026-10-20 -subtitle: How to build an app like p2 from nothing on Chord v2 and pi-durable, one object at a time, so that new plugin code takes over while the agent keeps running -stats: - - 11 | layers in the rebuild, L1 to L11 - - ~1,300 | lines built so far (L1 contracts) - - 6–7k | target size of Chord v2, without tests - - 0.004 ms | Delta's apply per small change -note: | - This guide teaches the **designed** API of Chord v2 (`packages/chord/src/next`, export `@earendil-works/chord/next`) - from `packages/chord/docs/next/design.md` and the newer decision log `packages/chord/docs/next/decisions.md`, which wins - where the two differ. Most of it is **not built yet**: a first kernel of about 10k lines is frozen as a reference and - Chord v2 is being rebuilt layer by layer; only the L1 contract layer exists (`src/next/contracts/`, commit - `7d2bb8ea8`). Code in this guide is therefore the target, not something you can run today. Every chapter says what is - built. API names are working names. Notes marked *review* are points that are undecided, or where the sources - disagree; they name the files. The reader is assumed to know pi-durable; Chord is explained from the start. -colophon: Chord v2 · a guide · first edition · 2026-10-20 · based on design.md (target design v2), redesign-decisions.md (review rounds 2 and 3), chord-next-walkthrough.md, layer-1.md, contracts/README.md, the p3 functionality map, questions.md and the pi-durable README ---- - -# Before we begin {.preface} - -It is late in the evening. Mario is on his phone, connected to his laptop at home through the Radius relay. In session -S1 the agent is twenty minutes into a refactoring; right now it is running a test suite through `bash`. A colleague, -Ann, follows the session read-only from her own browser. Mario notices that the notes panel sorts notes the wrong way -round. He fixes one line in the notes plugin on the laptop and presses **Reload**. - -What should happen is easy to say. The notes panel on both screens shows the new order. The test run keeps running. The -agent's turn continues as if nothing happened. Ann's view does not flicker into an error. Nothing that was written is -lost, and no process restarts. - -The obvious ways to get there fail. **Restarting the processes** loses the turn in flight, the open terminal and every -connection; pi-durable would resume the turn, but the test run and the connections are gone. **Hot-swapping modules** -in place, as front-end tooling does, leaves the old code alive in closures, timers, event listeners and sockets that -nobody knows about: the old sort function keeps running inside a callback someone stored, a file watcher opened by the -old code is never closed, and two versions of the plugin now write the same data without any order. - -Chord v2 is the runtime that makes the first outcome normal. It does so by being strict about one thing: **every piece -of state, every open OS handle and every piece of running work has exactly one owner, an object, and objects only -declare what they need.** Chord itself activates, connects, reloads and ends them. Because Chord knows every timer, -watcher, transaction and reference an object has, it can swap the object's code while its data stays, restart only -what touches the operating system, and route calls between browser tabs, the server and per-session worker processes -without the plugin author writing any of that. - -This guide teaches Chord v2 by building an app called **Studio**: a small version of p2 (Pi Mobile), with a server, -one worker process per session, browser tabs as clients, and an agent from pi-durable on each worker. The first -chapters grow one plugin, `@studio/notes`, until it uses every concept; the last three chapters assemble the rest of a -p2-like app from the same parts. - -:::review | Review: what exists today -Nothing in this guide runs yet except the contract layer. `packages/chord/src/next/contracts/` (L1 part 1: `remote`, -`local`, `method`, `evolve`, `checkAccepts`, the schema helpers) is built and reviewed. The rest of L1 (inputs, -argument checks, principals, errors) is a draft spec (`packages/chord/docs/next/layer-1.md`). The frozen kernel in -`src/next/*.ts` implements an older version of the design (host values, a development freeze, JSONL storage, -descriptions on the wire); do not read it as the target. Build on the design, not on the frozen code. -::: - -## Ten things to know - -:::findings -1. **An app is a tree of hosts.** One server, any number of workers it starts on demand, any number of clients (tabs, - TUIs) that connect to it. Clients reach workers only through the server, which relays. -2. **Objects are the unit of everything.** An object type is a contract (name, input, placement, members, version); an - object is one instance per input and caller. The first `get` creates it, later gets share it, the last release ends - it. -3. **Factories declare, Chord activates.** A factory is synchronous: it declares documents, references, derived - values, OS handles, routines and jobs through `self`, and returns the members. Chord opens, binds and starts them, - and undoes them in reverse. -4. **Links stamp who is calling.** A caller's data never says who it is; the link it arrived on does. Each caller gets - its own object and the variant of members it may use. -5. **Shared data lives once, behind views.** A `local` shared store holds the data; per-caller `remote` views expose - what each caller may see. Only stable identities persist documents. -6. **State changes only in transactions.** Documents are JSON values written in `self.tx`; commits are atomic for - every viewer and travel as small operations, never as whole values. -7. **Failures look like disconnects.** A reference survives link drops, restarts and failures: calls park or fail, - values stay readable, and Chord retries by itself. There are no request ids. -8. **Everything long-lived has a declared form.** Standing behaviour is a routine, work that outlives a call is a job, - input from a person is a flow, behaviour from other plugins comes through a registry. Nothing else may hold a timer - or a callback. -9. **Reload swaps pure objects in place and restarts native ones.** Objects without OS handles get the new code with - their data and running work intact; objects with OS handles close and reopen them. A broken change leaves the whole - host on its old code. -10. **pi-durable runs inside a core object.** The agent harness is a host-lifetime object on each worker; models, - environments and tools reach it through registries, and tools reach Chord through `chord(fn, context)`. -::: - -## Words used throughout - -A **host** is one participant of the app: a process, a browser tab or a Durable Object, identified by a logical -identity and an **incarnation** that changes on every start. A **link** connects two hosts. The **server** is the one -addressable root host; a **worker** is a host the server starts for one session; a **client** is a tab or TUI, not -addressable. A **plugin** is a directory with a `package.json`; its code for one kind of host is a **facet**. **Base -code** is the app's own code (Chord calls its facet the core facet); it restarts the hosts when it changes, while -plugins reload. An **object type** (or **contract**) describes objects; an **object** is one instance; its **input** is -its key among objects of that type, and its **placement** computes from the input on which host it lives. A **factory** builds an -object; **`self`** is what a factory declares through. A **principal** is who calls, stamped by the link; a -**variant** is the set of members a principal gets. A **reference** is what `get` returns. A **readable** is a value -plus its ordered changes. A **document** is a named JSON value an object owns, written in a **transaction**. A -**derive** is a value computed from readables. A **native** is an OS-level value (a socket, a watcher, a process) -opened with `self.open`; an object with natives is a **native object**, one without is a **pure object**. A -**routine** is standing behaviour, a **job** is owned work that outlives its call, a **flow** is input into running -work. A **registry** is an owner's extension point, filled with **contributions**. **`ctx`** is what every call -receives: cancellation, caller, call-scoped references. A **reload** swaps a plugin's code inside running hosts; a -**restart** ends a process; a **revision** names the complete code set the hosts run. - -:::note | Reading the code in this guide -Imports are shown once per file and then omitted. `obj`, `str`, `arr` and friends are Chord's schema helpers (they -return ordinary TypeBox schemas). Helper functions that belong to the app (`authenticate`, `watchDir`, `uuidv7`) are -named and not shown. Where the design gives no name, the guide says it uses a working name. -::: diff --git a/packages/chord/docs/next/explainer/src/01-shape.md b/packages/chord/docs/next/explainer/src/01-shape.md deleted file mode 100644 index db18b5cfe..000000000 --- a/packages/chord/docs/next/explainer/src/01-shape.md +++ /dev/null @@ -1,246 +0,0 @@ - -# The Shape of an App {#shape lede="Which processes a Chord app has, how they find each other, and the start code that connects them."} - -Before any plugin exists, Studio needs processes and connections between them. In Chord each participant is a host, -and the app's own start code creates every host, connects them and decides how workers start. Chord deliberately does -none of that itself: a laptop app forks child processes, a test runs everything in one process, a Cloudflare -deployment routes through the edge. This chapter writes Studio's start code: `server.ts`, `worker.ts`, the browser -entry, and a test setup. - -## Three roles - -A Node app on Chord has one shape: - -:::flow wide | The three roles. Clients connect to the one server; the server starts workers and relays between them. -row "Clients: not addressable" muted -box tab1 "browser tab" "owner, local" -box tab2 "browser tab" "guest Ann" -box tui "TUI" "owner" -row "Server: exactly one" accent -gap -box server "server" "`Server()`" accent -gap -row "Workers: one per session, on demand" blue -box w1 "worker" "`Session({ id: \"S1\" })`" blue -box w2 "worker" "`Session({ id: \"S2\" })`" blue -box w3 "worker" "`Session({ id: \"S3\" })`" dashed -arrow tab1 -> server "WebSocket" -arrow tab2 -> server "WebSocket" -arrow tui -> server "WebSocket" -arrow server -> w1 "IPC" -arrow server -> w2 "IPC" -note "A worker for S3 does not exist until something asks for an object that lives there." -::: - -| Role | Addressable | Who it is comes from | Lifecycle | -| --- | --- | --- | --- | -| client | no | the authentication of its link | connects by itself | -| server | yes, exactly one | its own declaration | runs | -| worker | yes, many, by a typed address | stamped by the server, its parent | started on demand by the app | - -The **server** holds what all sessions share (the session list, access, settings) and starts workers. A **worker** -runs one session: its agent, its files, its terminal connections. **Clients** are browser tabs and TUIs. Nobody can -send a request *to* a client, so a client can only serve objects that nobody outside it needs; chapter IV names them -`local` types. - -Things that look like hosts but are not: a code mode sandbox (a worker thread inside a worker, owned by an object), a -remote environment's daemon (it speaks its own protocol, owned by an object), the TUI's sandbox behind its launcher. -A detached process such as a terminal host could become another kind of child of the server. - -## Host kinds and addresses - -Every addressable host has a typed address. The app declares its **host kinds** once, in base code: - -```ts -// studio/src/hosts.ts -import { hostKind, obj, str } from "@earendil-works/chord/next"; - -export const Server = hostKind("server"); -export const Session = hostKind("session", obj({ id: str() })); - -Server(); // the address of the server -Session({ id: "S1" }); // the address of the worker of session S1 -``` - -There are no host-name strings anywhere in app code. Chord serializes addresses internally; your code only calls the -kinds. An object type names its host with a placement function from its input to an address (chapter II), so the -type checker knows that `Session({ id: session })` needs a `session` field. - -## Links - -A **link** is an ordered, two-way channel of messages between two hosts; it only delivers. Chord ships three: a memory -pair for tests, IPC for a Node child process forked with `serialization: "advanced"`, and WebSocket. All three carry -the same binary message format and the same flow control (chapter XVII). Recovery policy belongs to the link adapter: -IPC needs no heartbeat, a WebSocket through a relay needs one, because TCP does not notice that a laptop went to sleep. - -Each link has a **direction**. A host has at most one link to its parent (`connect`) and any number of links from -below (`accept` for clients, `route` for workers). Requests arriving on a link act as the principal the link was given; -chapter III explains why that matters. - -## The server - -```ts -// studio/src/server.ts -import { createHost, ipcLink, webSocketLink } from "@earendil-works/chord/next"; -import { fork } from "node:child_process"; -import { WebSocketServer } from "ws"; -import { Server, Session } from "./hosts.ts"; -import { coreServerFacet } from "./core/server.ts"; -import type { P } from "./principal.ts"; - -const server = createHost

({ - address: Server(), - principal: { kind: "server" }, // who this host's own code acts as - storage: await openServerStorage(dataDir), // persisted documents (chapter V); memory if omitted - onError: (e) => { if (e.kind === "storage_failed") process.exit(75); else log(e); }, -}); - -await server.load([ - { name: "@studio/core", facet: coreServerFacet(config) }, // base code, under a fixed name - ...(await loadPlugins("server")), // [{ name, facet }] from the plugin loader -]); - -const wss = new WebSocketServer({ server: http, path: "/chord", maxPayload: 64 * 1024 * 1024 }); -wss.on("connection", (ws, req) => { - const principal = authenticate(req); // app code: cookie, loopback, device token - if (!principal) return ws.close(4401); - server.accept(webSocketLink(ws), { principal }); -}); - -server.route(Session, { - authorize: (p, { id }) => - p.kind === "owner" || p.kind === "server" || (p.kind === "guest" && p.session === id), - resolve: async ({ id }, { signal }) => { - const principal: P = { kind: "worker", session: id }; - const child = fork("worker.ts", [], { serialization: "advanced", signal }); - child.send({ id, principal, init: await initFor(id, signal) }); // the app's own start message - return { link: ipcLink(child), principal }; // Chord stamps `principal` on it - }, -}); -``` - -Each call has one job: - -| Call | Meaning | -| --- | --- | -| `createHost

({ address?, principal?, ...options })` | with an address: an addressable host whose own code acts as `principal`; without: a client. `P` is the app's principal type | -| `load([{ name, facet }])` | the facets this host runs, each with its plugin's `package.json` name; duplicates and invalid names are refused. Base code passes its core facet the same way | -| `accept(link, { principal })` | a link from a client; every request on it acts as `principal` | -| `route(Kind, { authorize, resolve })` | how to reach hosts of a kind. `authorize(principal, address)` runs before anything starts; `resolve(address, { signal })` returns `{ link, principal }` | -| `connect(link, { principal })` | the link to the parent, at most one, replaced after a reconnect; `principal` says who the parent is | -| `root(fn, { principal?, signal? })` | a call scope for code outside objects (a CLI command): gives `fn` a `ctx` (chapter VI) | - -Chord has no `listen`. Entry points differ too much: a loopback WebSocket server, a tunnel, the Radius relay -delivering channels, a Durable Object edge, a memory link in a test. Each entry point authenticates and calls -`accept`. Radius, for example, is one more line: - -```ts -radius.onChannel((ws) => - server.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); -``` - -## The worker - -```ts -// studio/src/worker.ts -const { id, principal, init } = await startMessage(process); // app code: the first IPC message -const worker = createHost

({ - address: Session({ id }), - principal, // { kind: "worker", session: id } - storage: await openSessionStorage(init.dataDir, id), // one storage per session -}); -await worker.load([ - { name: "@studio/core", facet: coreWorkerFacet(init) }, // serves the start data (chapter IX) - ...(await loadPlugins("worker")), -]); -worker.connect(ipcLink(process), { principal: { kind: "server" } }); -``` - -## A browser tab - -```ts -// studio/web/main.tsx -const client = createHost

(); // no address: a client -await client.load(await loadPlugins("web")); -client.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }); -const notes = await client.get(Notes, { session: "S1" }); // tab → server → worker S1 -``` - -## The first `get`, step by step - -The tab asks for the notes of S1. No worker for S1 runs yet. - -:::sequence animate | The first get for an object on a worker that is not running. The server authorizes, the app starts the worker, Chord stamps the link. -actor tab "browser tab" "owner" -actor server "server" accent -actor app "route(Session)" "app code" muted -actor worker "worker S1" blue -tab -> tab: placement: Session({ id: "S1" }) -tab -> server: hold Notes { session: "S1" } -server -> server: not mine: route(Session) -server -> app: authorize(owner, { id: "S1" }) -app --> server: true -server => app: resolve({ id: "S1" }) -app -> worker: fork + start message -worker -> worker: createHost, load, connect -app --> server: { link, principal: worker S1 } -server => worker: hold (stamped: owner) -worker -> worker: placement check, run factory -worker --> server: bound: snapshot -server --> tab: bound: snapshot -::: - -`resolve` runs once per address while its link lives. When the link drops (the worker crashed or the supervisor -stopped it) and references to that worker still exist, Chord calls `resolve` again by itself, with a per-address -backoff, so a tab that only watches recovers without a click. A worker start that already produced a link finishes -even if the request that triggered it went away. - -**Incarnations** make restarts visible. Chord mints a fresh id (a uuidv7) on every host start and sends it in the link -handshake; the other side sees a restart as a new incarnation of the same identity, and references to it re-hold -(chapter VI). - -## Tests: everything in one process - -The same code runs in a test with memory links. `resolve` builds the worker host in memory and returns one end of the -pair: - -```ts -server.route(Session, { - authorize: () => true, - resolve: async ({ id }) => { - const [up, down] = memoryLinkPair(); - const principal: P = { kind: "worker", session: id }; - const worker = createHost

({ address: Session({ id }), principal }); - await worker.load([{ name: "@studio/core", facet: coreWorkerFacet({ id, dataDir }) }, notesWorker]); - worker.connect(up, { principal: { kind: "server" } }); - return { link: down, principal }; - }, -}); -``` - -Memory links deliver the same encoded messages asynchronously, so a test runs the production protocol path. - -## What stays app code - -Starting processes, the start message, authentication, supervising and stopping idle workers are app code. Chord -provides one readable for the last one: `host.activity`, with `{ references, calls, jobs, lingering }` (viewers, calls -in flight, unsettled jobs, objects kept warm after their last reference). Studio's core reports it to the server -together with the agent's running tasks, and the server's supervisor stops a worker when everything has been zero for -five minutes, as p2 does. Chord counts; it has no idle policy. - -## Host options and timeouts - -`createHost` takes the options introduced later in this guide: `storage` (V), `principalKey` and `revokedBy` (III, -VII), `telemetry` (XVII) and `onError`. `onError` receives background errors and `{ kind: "storage_failed" }`; every -app must handle the latter, because Chord never ends the process. Studio exits with its own code so its supervisor -restarts it. - -Timeouts are host defaults with per-thing overrides; the full table is in the appendix. The two you meet first: -`readyTimeout` (60 s: how long an activation and an awaited `get` wait) and the park deadline (60 s: how long a call -waits through a link drop or a restart before failing `unavailable`). - -:::note | Built so far -Nothing in this chapter is built in Chord v2 yet: hosts, links and routes are layers L6 and L7. The frozen kernel has -working versions with the same shape (`createHost`, `connect`, `accept`, `route`, `load`), which the rebuild -simplifies. -::: diff --git a/packages/chord/docs/next/explainer/src/02-first.md b/packages/chord/docs/next/explainer/src/02-first.md deleted file mode 100644 index 5c01e5213..000000000 --- a/packages/chord/docs/next/explainer/src/02-first.md +++ /dev/null @@ -1,187 +0,0 @@ - -# A First Object {#first-object lede="A plugin, a contract, a factory, and a browser tab that calls it on another process."} - -Studio's first feature is session notes: a list of short notes per session, shown in a panel, that anyone allowed can -add to. Several places need "the notes of S1": the owner's tab, a guest's tab, later a download route and an agent -tool. If each built its own copy, the copies would drift apart and nobody would know when to clean them up. In Chord -there is **one object** for "the notes of S1", everyone asks for it by its type and input, and Chord routes them all to it. - -## The plugin's files - -A plugin is a directory with a `package.json`. Its `name` (npm style, globally unique and stable) is the plugin's -identity: Chord finds its stored documents, reports its errors and orders its contributions by it, never by a -directory name or load order. - -| File | Runs on | Holds | -| --- | --- | --- | -| `package.json` | read by the loader | `"name": "@studio/notes"` | -| `contract.ts` | every host that imports it | object types: names, shapes, method signatures; no running code | -| `worker.ts` | every session worker | the worker facet | -| `server.ts` | the server | the server facet (later: the download route) | -| `web.tsx` | browser tabs | the web facet: the panel | - -## The contract - -An object type is a contract: a name, an **input** (the object's key among objects of this type), a **placement** -(on which host it lives, computed from the input), the **members** (state you can read and methods you can call) and a **version**. - -```ts -// notes/contract.ts -import { arr, int, method, obj, remote, str, type Static } from "@earendil-works/chord/next"; -import { Session } from "@studio/core/hosts"; - -export const Note = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int() }); -export type Note = Static; - -export const Notes = remote("notes", { - version: 1, - input: obj({ session: str() }), // the key: { session: "S1" } - placement: ({ session }) => Session({ id: session }), // input → the host's address - variants: { - editor: { - state: { notes: obj({ items: arr(Note) }) }, - methods: { - add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), // returns the new id - remove: method(obj({ id: str() })), // no result - }, - }, - }, -}); -``` - -`remote` means objects of this type can be reached from any host, across links (chapter IV introduces `local`). The -schema helpers (`obj`, `str`, `int`, `arr`, `map`, `opt`, `lit`, `oneOf`, `union`, `bool`, `num`, `nul`, `bytes`, -`unknown`) return ordinary TypeBox schemas; short options such as `{ min, max }` bound lengths or values. Schemas must -describe data: no callbacks, no class instances, no custom codecs. - -`method()` has four forms: `method()` (no arguments, no result), `method(args)`, `method(args, result)` and -`method(undefined, result)`. - -A contract contains no code that runs, so every host can import it. When you declare it, Chord materializes its -description and computes a **fingerprint**, a hash of its input, variants and members. Two hosts compare contracts by -version and fingerprint (chapter XVI). Placement is not part of the fingerprint. - -## Placement - -The input says *which* object; `placement` says *where* it lives. The caller runs the placement function on the input and sends the resulting address with its request; the server -routes by it. The provider runs the function too and refuses an object that is not its own with `invalid/placement`. -So a worker for S2 never builds "Notes for S1", even if a caller has an outdated rule. Because placement is a typed -function, a `Session` placement on a type whose input has no `session` field does not compile. - -Inputs are **normalized** before anything else: defaults are applied, unknown fields removed, and a canonical key -computed. `{ session: "S1" }` and `{ session: "S1", extra: 1 }` are the same object. - -## The facet and the factory - -A **facet** is a plugin's setup for one kind of host. It runs once when the plugin loads on that host and tells the -host which object types it serves: - -```ts -// notes/worker.ts -import { defineState } from "@earendil-works/chord/next"; -import { defineFacet } from "@studio/core"; // Studio's defineFacet: principals typed as Studio's P (chapter III) -import { Note, Notes } from "./contract.ts"; - -const NoteList = defineState("notes.list", { // a document (chapter V); in memory for now - schema: obj({ items: arr(Note) }), - version: 1, - initial: () => ({ items: [] }), - checkpointWhen: () => true, -}); - -export default defineFacet((host, self) => { - host.serve(Notes, ({ session }, principal, self) => { - const list = self.state(NoteList); // THE note list of this object - return { - variant: "editor", - notes: list, // a document handle is a readable: expose it as state - add: ({ text }, ctx) => - self.tx(async (tx) => { - const id = uuidv7(); - (await tx.draft(list)).items.push({ id, text, by: "someone", at: Date.now() }); - return id; - }, ctx), - remove: ({ id }, ctx) => - list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), - }; - }); -}); -``` - -Read `host.serve(Notes, factory)` as "this host builds `Notes` objects like this". The factory receives the -normalized input, the caller's principal (chapter III) and `self`, through which it declares what the object needs. -It returns the members of one variant, tagged with `variant`. - -**Factories declare; Chord activates.** The factory is synchronous and has no side effects. It does not await, open -files or start timers. `self.state(NoteList)` does not load anything: it declares that the object has this document -and returns a **handle** that stays valid for the object's whole life. Chord then performs the activation: it loads the -document, binds references, opens OS handles, starts routines, and only then lets calls in. The rule exists for -reload: a reload runs the factory again, and anything the factory did itself would happen twice. - -## Calling it - -```tsx -// notes/web.tsx -export function NotesPanel({ session }: { session: string }) { - const notes = useRef(Notes, { session }); // app-side hook over host.get / release - const items = useReadable(notes?.notes)?.items ?? []; - return ( -

- {items.map((n) =>

{n.text}

)} - notes?.add({ text })} /> -
- ); -} -``` - -Underneath the hooks is the reference API: - -```ts -const notes = await client.get(Notes, { session: "S1" }); // live and hydrated once awaited -notes.notes.value.items; // the current list -notes.notes.subscribe((v) => render(v.items)); // every committed change -const id = await notes.add({ text: "check the flaky test" }); -notes.release(); // the object may end now -``` - -An awaited reference is **live and hydrated**: every state member has a value. State members are **readables**: -`.value` is the current value, `subscribe` is called after every commit. The first `get` for `{ session: "S1" }` runs -the factory; every later `get` for the same input and caller shares that object. The object ends when the last -reference is released (a type may keep it a little longer with `linger`). - -:::sequence animate | A call to add. Arguments are checked once at the provider; the commit reaches the viewer as one small change. -actor tab "browser tab" -actor server "server" muted -actor worker "worker S1" blue -actor obj "Notes { S1 }" accent -tab -> server: call add { text } -server -> worker: relay (header only) -worker -> obj: strip + validate args -obj => obj: self.tx: push note, commit -obj --> worker: result: id -worker --> server: state: ops of the commit -worker --> server: result -server --> tab: state, then result -::: - -## Two rules for members - -**Methods return data, never objects.** A method that wants to point you to another object returns its address -(`{ session: "S2" }`), and you `get` it yourself. That keeps every reference owned by someone who releases it. - -**No callbacks as arguments.** A callback carries code, and everything it closes over, from one object into another. -Suppose an inbox object passed `onFile` to another object, which stored it. After a reload of the inbox's plugin, the -other object still calls the *old* `onFile`; after the inbox ends, it calls into an object whose file watcher is -already closed. Chord forbids this by construction: members are data (JSON plus bytes). Behaviour from elsewhere goes -through registries (chapter XIII), multi-step work through jobs (XI), input from a person through flows (XII). - -## What is wrong with this version - -This first version works for one person. It has two problems, which the next two chapters fix. Everyone gets the -`editor` variant, including a read-only guest. And, less obvious, the owner and a guest do not see the same notes: as -chapter III shows, a remote object exists per caller, so each caller's `Notes` object has its own `NoteList`. - -:::note | Built so far -`remote`, `method`, the schema helpers, `Static` and fingerprints are built (`src/next/contracts/`). Serving, `get`, -references and documents are layers L3 to L5. -::: diff --git a/packages/chord/docs/next/explainer/src/03-principals.md b/packages/chord/docs/next/explainer/src/03-principals.md deleted file mode 100644 index 52a4c1064..000000000 --- a/packages/chord/docs/next/explainer/src/03-principals.md +++ /dev/null @@ -1,190 +0,0 @@ - -# Who Is Calling {#principals lede="Principals: how Chord knows who calls, why a caller cannot lie about it, and how the answer picks the object and its members."} - -The owner may add and remove notes. Ann, a read-only guest, may only read them and should not even see who wrote -what. The object must therefore know who is calling, and the caller must not be able to claim to be someone else. In -Chord the answer is a **principal**, a small JSON value the app defines and the links stamp. - -## The app's principal type - -Chord defines no principal kinds; the app does. Every variant has a `kind`. Studio uses p2's set: - -```ts -// studio/src/principal.ts -export type P = - | { kind: "owner"; access: "local" | "relay" | "device"; device?: string } - | { kind: "guest"; session: string; mode: "read" | "write"; grant: string; revision: number } - | { kind: "server" } - | { kind: "worker"; session: string }; -``` - -| `kind` | Who | -| --- | --- | -| `owner` | the person running the app, on loopback, through the relay, or on a paired device | -| `guest` | someone invited into one session, with a mode and the grant (and its revision) that let them in | -| `server` | the server's own code calling a worker | -| `worker` | a session worker calling the server | - -## Links stamp principals - -A principal never comes from a request's data. The server authenticates a client once, when its WebSocket connects, -and stamps the result on the whole link (`server.accept(link, { principal })`). The server tells Chord who each worker -is when it resolves it (`return { link, principal }`). When the server relays a client's request to a worker, Chord -writes the client's principal into the forwarded request; a principal field coming from a client is always -overwritten, and only a trusted parent link may carry one. - -Where principals come from is app code. Studio's sharing feature creates a share link per session and mode; a guest -who opens it gets a guest record (grant `g1`) and a cookie; `authenticate(req)` turns the cookie into -`{ kind: "guest", session: "S1", mode: "read", grant: "g1", revision: 1 }` on every new connection. Chord never sees -cookies or grants, only the finished principal. - -## The caller is part of the identity - -A `remote` object exists once per **(type, input, principal key)**. The **principal key** is a string computed from -the principal; by default its canonical JSON (sorted keys). The app may pass its own `principalKey` to `createHost`, -and must pass the same function on every host, or identities split silently. - -```ts -const principalKey = (p: P) => - p.kind === "owner" ? `owner:${p.access}:${p.device ?? ""}` - : p.kind === "guest" ? `guest:${p.grant}:r${p.revision}` - : p.kind === "worker" ? `worker:${p.session}` - : p.kind; -``` - -The design wants the key to include how the client is connected and the grant's revision, so that two access paths -never share objects and a changed grant never reuses the old grant's object: - -| Caller | Principal key | `Notes { S1 }` object | -| --- | --- | --- | -| owner, tab A, local | `owner:local:` | 1 | -| owner, tab B, local | `owner:local:` | 1 (shared with tab A) | -| owner through the relay | `owner:relay:` | 2 | -| guest g1, revision 1 | `guest:g1:r1` | 3 | -| guest g1 after a change to write mode, revision 2 | `guest:g1:r2` | 4 (new) | - -## Variants - -The contract lists named sets of members; the factory receives the principal and returns the variant that principal -may use: - -```ts -// notes/contract.ts (version 1, final form for now) -variants: { - editor: { - state: { notes: obj({ items: arr(Note) }) }, - methods: { add: method(obj({ text: str({ min: 1, max: 2000 }) }), str()), remove: method(obj({ id: str() })) }, - }, - reader: { - state: { notes: obj({ items: arr(PublicNote) }) }, // no `by`: authors are hidden - }, -}, -``` - -```ts -// notes/worker.ts -host.serve(Notes, ({ session }, principal, self) => { - if (canWrite(principal, session)) return { variant: "editor", notes: …, add: …, remove: … }; - return { variant: "reader", notes: … }; -}); -``` - -A reader has no `add` at all. Chord refuses a call to a member that is not in the caller's variant, so `add` needs no -check of its own. Three rules keep variants compatible: a member name has the same kind (state or method) in every -variant that has it; a method has the same arguments and result in every variant; only state schemas may differ per -variant (the reader's notes have no authors). `remote(...)` enforces them at the type level, and at runtime for -contracts that arrive as data. - -## How a facet knows `P` - -`P` is Studio's type, not Chord's, and every host is created with it (`createHost

`). A plugin's facets are written -and compiled separately, then handed to `host.load` as values, so the type checker cannot infer `P` from the host into -the factory's `principal`. Studio's base code therefore exports its own `defineFacet`, already bound to `P`, and every -facet of every plugin imports that one instead of Chord's: - -```ts -// studio/src/facet.ts (base code, exported as @studio/core) -import { bindFacet } from "@earendil-works/chord/next"; -import type { P } from "./principal.ts"; - -export const defineFacet = bindFacet

(); // a facet's principals are Studio's P -export type { P, Revocation } from "./principal.ts"; -``` - -```ts -// notes/worker.ts (server.ts, web.tsx and tui.ts import the same defineFacet) -import { defineFacet } from "@studio/core"; - -export default defineFacet((host, self) => { - host.serve(Notes, ({ session }, principal, self) => { // principal: P - if (principal.kind === "guest" && principal.mode === "read") return { variant: "reader", notes: … }; - … - }); -}); -``` - -The import is cheap: `@studio/core` is a host package plugins already import at runtime (the contract imports `Session` -from it), `P` is a type-only import, and `bindFacet` is a one-line function. The host checks the pairing: `load` -accepts only facets typed for its own `P`, so a facet compiled against another app's principal type, or one built with -Chord's untyped `defineFacet`, does not compile into Studio's start code. Client facets (`web.tsx`, `tui.ts`) use the -same `defineFacet`; they rarely see a principal, because clients serve only `local` types. - -None of this is checked at runtime, and none of it needs to be: the principal a factory receives is the one the link -stamped. - -:::review | Review: binding facets to the app's principal type (open) -The design shows `defineFacet((host, self) => …)` without saying how `P` reaches the factory. The frozen kernel takes -it as an explicit type parameter, `defineFacet

(…)`, defaulting to `{ kind: string }` -(`packages/chord/src/next/facet.ts`), and the playground writes `defineFacet(…)`. The pre-bound -`defineFacet` above (`bindFacet` is a working name) is the owner's preferred direction, recorded as an open point in -`packages/chord/docs/next/layer-1.md` §4. -::: - -## Narrowing on the caller's side - -Owner and guest run the same `web.tsx`, and which variant they get is known only at runtime. The types force every -caller to narrow before using variant members: - -```ts -const notes = await client.get(Notes, { session: "S1" }); -notes.add({ text: "hi" }); // type error: `add` exists only in "editor" -if (inVariant(notes, "editor")) await notes.add({ text: "hi" }); -expectVariant(notes, "editor"); // or assert (throws) -function Editor({ notes }: { notes: RefIn }) { … } -``` - -The protection is not the type checker; it is Chord refusing the call at the provider. A client that ignores the -types gets the same error. - -## Admission before a worker starts - -Principals also decide whether a caller may reach a host at all. `route(Session, { authorize })` runs before Chord -resolves an address, so a guest of S1 asking for anything in S2 is refused at the server, and no worker starts for -them. Finer authority lives in the per-principal view's methods; there is no asynchronous "may this caller activate -this object" guard. An object whose address carries authority (say, a login flow by id) may exist; what a caller can -see and do is decided by the variant its factory picks. - -## Inside methods - -`ctx.principal` is the caller of the current call. In Studio's notes the editor's `add` uses it to record the -author: - -```ts -add: ({ text }, ctx) => store.add({ text, by: authorOf(ctx.principal) }, ctx), -``` - -Where principals act, in summary: - -| Purpose | Where | -| --- | --- | -| identity: which object a caller gets | principal key | -| variant: which members | the factory | -| admission: whether a host is reachable | `route.authorize` | -| attribution and fine checks | `ctx.principal` in methods | -| revocation | `revokedBy` (chapter VII) | - -## The bug from chapter II - -Now the bug is visible. The owner's `Notes { S1 }` and Ann's `Notes { S1 }` are two objects (rows 1 and 3 above), and -each declared its own `NoteList`. The owner's notes are not in Ann's list. Per-caller objects must not hold the data; -they must be views over something that exists once. That is the next chapter. diff --git a/packages/chord/docs/next/explainer/src/04-store.md b/packages/chord/docs/next/explainer/src/04-store.md deleted file mode 100644 index 44d3739ff..000000000 --- a/packages/chord/docs/next/explainer/src/04-store.md +++ /dev/null @@ -1,132 +0,0 @@ - -# One Store, Many Views {#shared-store lede="Local types, and the pattern every Chord plugin uses: shared data once in a store, per-caller views in front of it."} - -Every caller has its own `Notes` object, which is what makes per-caller variants possible. But data must exist once. -The fix is the most common pattern in Chord: a **shared store**, a `local` object that holds the data and does the -heavy work once, behind thin per-principal **views**, the `remote` objects callers get. - -## `local` types - -```ts -// notes/contract.ts -export const NoteStore = local("notes.store", { - version: 1, - input: obj({ session: str() }), - members: { - notes: obj({ items: arr(Note) }), // state: a schema - public: obj({ items: arr(PublicNote) }), - add: method(obj({ text: str(), by: str() }), str()), // methods: method(...) - remove: method(obj({ id: str() })), - }, -}); -``` - -A `local` contract has flat `members` and no variants. The differences: - -| | `remote` | `local` | -| --- | --- | --- | -| reachable from | any host, across links | its own host only | -| identity | (type, input, principal key) | (type, input) | -| principals, variants | yes | no | -| factory | `(input, principal, self)` | `(input, self)` | -| arguments | stripped to the provider's schema and validated, for every caller | see the review note below | -| values on the same host | passed by reference, never copied | passed by reference, never copied | -| `ctx.principal` | the caller | none | -| may persist documents | no (chapter V) | yes | - -A local object has no principal: it is trusted code on its own host, acting for the host. So attribution ("who wrote -this note") is passed as data (`by`), by the view that knows who called. Clients serve only local types, because -nobody can address a client. - -## The pattern - -```ts -// notes/worker.ts -export default defineFacet((host, self) => { - host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); - const pub = self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); - return { - notes: list, - public: pub, // computed once, for every reader - add: ({ text, by }, ctx) => - self.tx(async (tx) => { - const id = uuidv7(); - (await tx.draft(list)).items.push({ id, text, by, at: Date.now() }); - return id; - }, ctx), - remove: ({ id }, ctx) => list.change((d) => { d.items = d.items.filter((n) => n.id !== id); }, ctx), - }; - }); - - host.serve(Notes, ({ session }, principal, self) => { - const store = self.get(NoteStore, { session }); // a reference, declared: held for the view's life - if (canWrite(principal, session)) return { - variant: "editor", - notes: store.notes, // the store's readable, exposed as-is - add: ({ text }, ctx) => store.add({ text, by: authorOf(principal) }, ctx), - remove: (args, ctx) => store.remove(args, ctx), - }; - return { variant: "reader", notes: store.public }; - }); -}); -``` - -:::flow wide | One store, three views. The redaction runs once in the store; both readers read the same value. -row "Per-principal views (remote)" blue -box ve "Notes { S1 }" "owner · editor" blue -box vr1 "Notes { S1 }" "guest Ann · reader" blue -box vr2 "Notes { S1 }" "guest Bob · reader" blue -row "Shared store (local)" accent -gap -box store "NoteStore { S1 }" "notes, public (a derive)" accent -gap -arrow ve -> store "self.get" -arrow vr1 -> store "self.get" -arrow vr2 -> store "self.get" -note "Authority is the view's: the reader variant has no add. The store does not check; it is trusted and acts for the host." -::: - -`self.get(NoteStore, { session })` in a factory declares a **reference** that the view holds for its whole life; Chord -binds it during activation and waits until the store is live (chapter VI). The view exposes the store's readables -directly. Owner, Ann and Bob now give three `Notes` objects, one `NoteStore`, one note list and one redacted list. - -Any heavy or shared part belongs in a store like this: a pi-durable watch, a file watcher, a sandbox, a search index. -Base code uses the same pattern with the same primitives; nothing about it is special to core. - -## Derived values, briefly - -`self.derive(name, sources, fn)` computes a value from readables and recomputes it when a source changes; viewers get -only the difference. The reader's list above is one. Chapter VIII covers derives in depth, including the one footgun -that matters: `stripAuthor` must return the *same* object for an unchanged note, or every commit costs the size of the -whole list. The tool for that is `mapShared`. - -## Arguments are checked at the provider - -Trusted code is not type-checked code: plugins are bundled with esbuild without type checks, and agents write -JavaScript plugins. So the provider checks arguments itself (decision R7): every method's arguments are stripped to -the **provider's own** schema (unknown fields removed) and validated with it, whoever the caller is, and never with a -schema a peer sent. Schemas are compiled lazily on first use and cached per schema object. Bytes are checked as leaves -(`instanceof Uint8Array` plus an optional length bound), never walked. Stripping copies only when a field was removed; -the caller's value is never mutated (proposal Q12, in the L1 draft). Results and state are not validated: they come -from the provider. - -On the same host nothing is copied: arguments and results are passed by reference. Values are **immutable by -convention** (Delta's rule, as in Durable): nobody mutates a value it received, and Chord neither copies nor freezes -values to enforce it. There is no development mode that would freeze them; one behaviour everywhere (R9). - -:::review | Review: are local arguments validated? (Q15, open) -Design §5's table says `local` arguments are "neither validated nor copied"; §3 and R7 say arguments are validated -"for every caller". Review round 3 raised this as Q15 (apply R7 to `local` too, or keep `local` as the one exception), -and `layer-1.md` lists it as open. Until it is decided, write local methods so that they do not depend on either -answer: trust the shape, but never trust content (lengths, ids) that came from a remote caller without the view -checking it. `chord-next-walkthrough.md` concept 6 still describes the older rule ("validated only when the caller is -not a host principal"), which R7 replaced. -::: - -## Choosing between the two - -Use a `remote` type for what callers address: views, things with per-caller authority. Use a `local` type for what -exists once per input on one host: stores, caches, owners of OS handles, and the core objects a host provides -(chapter IX). If a type is reached across a link, it is remote; if a type persists documents, it is local (or a -host-lifetime object, chapter V). diff --git a/packages/chord/docs/next/explainer/src/05-documents.md b/packages/chord/docs/next/explainer/src/05-documents.md deleted file mode 100644 index 2a25a1caa..000000000 --- a/packages/chord/docs/next/explainer/src/05-documents.md +++ /dev/null @@ -1,258 +0,0 @@ - -# Writing It Down {#documents lede="Documents and transactions: how an object stores data, changes it atomically, survives restarts and migrates it."} - -Notes must be stored, changed safely while others read them, sent to viewers as small changes, and survive a worker -restart. An object's data lives in **documents**: named JSON values the object owns, changed only in -**transactions**. The design is a port of pi-durable's store, so most rules will look familiar; this chapter states -them for Chord. - -## Defining a document - -```ts -// notes/worker.ts -const NoteList = defineState("notes.list", { - schema: obj({ items: arr(Note) }), - version: 1, - initial: () => ({ items: [] }), - persist: true, // stored; default: memory only - checkpointWhen: (_value, _ops, info) => info.deltasSinceBase > 64, -}); - -const Draft = defineState("notes.draft", { // one unfinished note per author - schema: obj({ text: str() }), - version: 1, - initial: () => ({ text: "" }), - keyed: true, // many per object, by key - persist: true, - checkpointWhen: () => true, // small: a full base per write -}); -``` - -`checkpointWhen` is required on every definition, with no default (R14, as in Durable since pi `5c1736f82`): it says -when a commit stores a complete base instead of a delta, which bounds how much is replayed on load. Small documents -checkpoint on every write; growing ones after a number of deltas. - -A document's root is always a JSON object (lists live in a field). Documents hold JSON only; bytes are a value and wire -type, not a document type. - -## Singleton and keyed - -```ts -host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); // THE note list of this object - const drafts = self.state(Draft); // all drafts of this object - … -}); -``` - -| | singleton (no `keyed`) | keyed | -| --- | --- | --- | -| how many | one per object | many per object, by a short key | -| loaded | at activation, kept in memory | on demand | -| read | `list.value` (synchronous) | `await drafts.get(key)`: the value or `undefined` | -| list | (it is one value) | `for await (const key of drafts.keys())` pages keys | -| expose as state | yes: the handle is a readable | no: expose a method or a singleton index | - -There is no live key list. A collection shown live is a singleton (an object holding a list or a map); a large -collection is a method returning pages. - -## Transactions - -All writes go through `self.tx(fn, ctx)`. Inside, `await tx.draft(handle, key?, { tags? })` returns a draft of the -current document, or a draft over `initial()` if it is absent. Drafts record operations (assignments, `push`, -`Object.assign`); asking for the same document twice in one transaction returns the same draft. `tx.retire(handle, -key?)` removes a document. There is no `tx.get`: reading is drafting, and a draft is an overlay over the current -value, so nothing is copied. A draft you do not change commits nothing, so reading never stores an absent document. - -```ts -saveDraft: ({ author, text }, ctx) => - self.tx(async (tx) => { (await tx.draft(drafts, author)).text = text; }, ctx), -``` - -Shorthands for one document: `handle.change(fn, ctx)` (keyed: `handle.change(key, fn, ctx)`) (loads the document, then runs the synchronous `fn` on -its draft) and `handle.retire(ctx)` (keyed: `handle.retire(key, ctx)`). - -### Four patterns - -Two decisions (R12 and R13, both as in Durable) shape how transactions are written. **Drafts are revoked when the -transaction settles**, so a transaction returns at most plain values, never something taken from a draft. And -**there is no existence check inside a transaction.** - -```ts -// 1. Return data: write in the transaction, read after the commit. -await self.tx(async (tx) => { (await tx.draft(list)).items.push(note); }, ctx); -return list.value.items.length; // the committed revision, or a newer one - -// 2. Remove if present: retire of an absent document is a no-op. -await drafts.retire(author, ctx); - -// 3. Create with tags if missing: the draft creates it, only if it changes. -await self.tx(async (tx) => { - (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; -}, ctx); - -// 4. Branch on existence: read first, outside the transaction. -if ((await drafts.get(author)) === undefined) { … } -``` - -### Durable's rule - -Inside the callback, only `await tx.*`. No other objects, no network, no models, no processes, no second `self.tx`, no -reads past `tx`. Breaking it is not an error but a hang: the object's line is held while the callback waits. Fetch -first; decide and write inside. - -```ts -// Wrong: a fetch inside the transaction holds the object's line. -await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title: await fetchTitle(url) }); }, ctx); -// Right: fetch, then write. -const title = await fetchTitle(url, ctx.signal); -await self.tx(async (tx) => { (await tx.draft(list)).items.push({ …, title }); }, ctx); -``` - -## What a commit does - -:::flow wide animate | The commit sequence, Durable's. Nothing is copied, frozen or revalidated; viewers get one batch per commit. -row "Inside self.tx" accent -box line "wait for the line" "per object" -box run "run the callback" "drafts record ops" accent -box prep "prepare" "Delta checks placed values" -box write "storage batch" "deltas or a base" accent -row "After the write" blue -box adopt "adopt" "pointer swap" -box local "notify locally" "once, with the ops" blue -box wire "send" "one batch per link" -box revoke "revoke drafts" "touching one throws" -arrow line -> run -arrow run -> prep -arrow prep -> write -arrow write -> adopt "stored" -arrow adopt -> local -arrow local -> wire -arrow wire -> revoke -note "An error or an abort before the storage write discards everything. Storage commits are serialized per host." -::: - -Lines are per object, never across objects; reads outside a transaction do not take the line. Subscribers on the same -host get the new value and the commit's operations by reference. Remote viewers get the commit's operations, never the -document again; chapter XVII shows how one commit touching several holds still arrives as one message. - -## Tags and deleting by tag - -Tags are typed tokens set when a document is created: - -```ts -export const SessionTag = defineTag("studio.session"); // base code, shared by every plugin -await self.tx(async (tx) => { (await tx.draft(drafts, author, { tags: [SessionTag(session)] })).text = text; }, ctx); -// when the session is deleted -await store.retireByTag(SessionTag, "S1"); // every document tagged S1 -``` - -The design names the operation `store.retireByTag(tag, value)` without saying who holds the store; session delete is -one of the app decisions still open (#18, chapter XIX). - -## Who may persist - -`persist: true` is allowed only in `local` objects and `lifetime: "host"` objects: identities without a principal. A -`remote` object's identity contains the principal key, which changes with the access path and the grant revision. Its -stored documents would scatter across access paths and be orphaned after a reconnect. The rule is enforced at the type -level (`self.state` in a remote factory accepts only non-persisted definitions) and at declaration ("persist needs a -stable identity: put the documents into a local shared store"). This is the second reason for the shared store. - -What survives a host teardown (a crash, a restart) is exactly what is persisted. In-memory documents live as long as -their object: they survive reloads and native restarts of a live object, and go when it ends. Everything else -(natives, derives, routines, references) is rebuilt on activation. - -## Facet documents - -A facet has its own identity (its plugin's name) and may hold documents too, for plugin-wide data such as -preferences. Objects the facet serves may read and write their facet's documents, and one transaction may span the -object's own documents and its facet's (lock order: own line, then the facet's). The facet never touches a served -object's documents. - -## Ownership across plugins - -No plugin reads or writes another plugin's documents, Chord or Durable. Data crosses plugins only through state and -methods. Data that core enforces lives in a format core owns and is handed to core through its API when the data is -created (as with Durable's `createConversation({ init })`), never written into core's documents. - -## Versions and migration - -A change of shape or meaning raises `version` and adds a pure `migrate(old, from)`: - -```ts -const NoteList = defineState("notes.list", { - schema: obj({ items: arr(NoteV2) }), // NoteV2 adds `pinned` - version: 2, - migrate: (old, from) => ({ items: old.items.map((n) => ({ ...n, pinned: false })) }), - initial: () => ({ items: [] }), - persist: true, - checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, -}); -``` - -Documents migrate on access by the new definition: singletons during activation (hydration counts as access), keyed -ones on their next access. The migrated base is persisted with the next commit; right after a bump, a read-only -transaction writes it. - -Chord does not walk values against schemas. Instead, each definition computes a **schema fingerprint** once -(descriptions and titles ignored), and every stored record keeps the fingerprint it was written with. On access Chord -compares them: the same version with a different fingerprint fails the object `broken` ("schema changed without a -version bump"). So every structural change needs a bump, an additive one with a trivial `migrate`. The one exception -to "no value checks" (R8): the result of `migrate` and of `initial()` is validated against the new schema once, when -produced. A failed check, a throwing `migrate`, or old code meeting a newer document refuses the access, keeps the -stored data and fails the owning object. A changed meaning without a schema change cannot be detected; that is on -you. - -## Storage - -Behind the store is a `DocumentStorage`, passed as `createHost({ storage })`; memory by default. The server has one -shared storage, each worker one for its session. - -```ts -interface DocumentStorage { - commit(writes: readonly DocumentWrite[]): Promise; // create/change/retire, atomic - find(address: DocumentAddress): Promise; - load(id: string): Promise; // version, value, deltasSinceBase - scan(query: { owner: string; definition: string }, limit: number, cursor?: string): - Promise<{ keys: string[]; next?: string }>; - byTag(tag: string, value: string, limit: number, cursor?: string): Promise<{ ids: string[]; next?: string }>; - mintId(): Promise; -} -``` - -Chord ships a memory storage and a SQLite storage (over `@earendil-works/chord/sqlite`, the driver layer that moved -from Durable; Node and Durable Object drivers). Storages check only metadata (incarnation, version, sequence, address -occupancy, fingerprint) and write the supplied delta or base; they never rebuild or validate content on a commit. -Chord documents never commit together with Durable transcripts: Chord has its own storage, not an adapter over -Durable's. Your own storage can prove itself with `registerDocumentStorageConformance(runner, name, withStorage)` from -`@earendil-works/chord/testing`. - -Keyed persisted documents that nobody reads, writes or watches for 10 minutes leave memory and reload on the next -access. A storage error ends the store and closes the host; `onError` receives `{ kind: "storage_failed" }` and the -app decides (chapter I). - -## Kinds of state - -| Kind | For | API | -| --- | --- | --- | -| document | data the object owns | `self.state`, `self.tx` | -| derive | computed values | `self.derive` (chapter VIII) | -| source readable | external data: Durable views, files, logs | value plus ordered changes; feeds derives | -| paged method | large or queryable collections | a method returning pages (chat history) | -| plugin database | full-text search, large blobs | a native opened by its owner (chapter IX) | - -App state lives in Chord documents. Only transcript-bound data, which must fork and rewind with a conversation, lives -in Durable. - -:::review | Review: does an absent singleton read `undefined` or `initial()`? (Q11, open) -Design §9 says "whether a document exists is read outside the transaction (`handle.value` / `handle.get` give -`undefined` for an absent one)" and also "a singleton that does not exist yet reads `initial()` through its handle; -retiring resets it to `initial()`". Review round 3 proposes: singletons always read `initial()`, only absent keyed -documents read `undefined`, and singleton existence needs no API. This guide follows the proposal (`list.value` is -never `undefined`); it is not decided. -::: - -:::note | Built so far -Documents, transactions and storages are layer L3, a port of Durable's session store; not started. The frozen kernel -has a working version with a JSONL storage, which the design rejected. -::: diff --git a/packages/chord/docs/next/explainer/src/06-references.md b/packages/chord/docs/next/explainer/src/06-references.md deleted file mode 100644 index b852916b5..000000000 --- a/packages/chord/docs/next/explainer/src/06-references.md +++ /dev/null @@ -1,185 +0,0 @@ - -# Holding On {#references lede="References, the object lifecycle and ctx: what a caller sees while objects start, fail, restart and move, and why nobody writes retry code."} - -A tab shows the notes of S1 for an hour. During that hour the laptop sleeps, the worker is stopped for idleness and -started again, the notes plugin reloads twice and the store's storage hiccups once. The panel should show the notes the -whole time, say "reconnecting" when it cannot reach them, and recover by itself. In Chord this is what a -**reference** does; plugin code contains no retry loops and no request ids. - -## Three ways to hold - -| Where | API | Released | -| --- | --- | --- | -| a factory | `self.get(Type, input)` | when the object ends | -| a call | `await ctx.get(Type, input, { state? })` | when the call ends | -| outside objects | `await host.get(Type, input, { signal, state?, parkTimeout? })` | `ref.release()` | - -Use `self.get` for what an object needs its whole life (the view's store), `ctx.get` for what one call needs, and -`host.get` only in UI code, tests and entry points. `{ state: false }` binds for methods only, without receiving state. -A reference keeps its object alive while held. - -## The object lifecycle - -```text -(get) → activating → live → ending → ended - └→ failed (backoff) → activating … -``` - -**Activating:** Chord binds the declared references and waits until they are live and hydrated, loads singleton -documents, opens natives (one after another, in declaration order), computes derives and starts routines. The object -goes live when all of that succeeded. Calls wait behind its **gate** until then. - -**Ending:** routines stop, the object's calls and jobs are aborted as their rules say, natives close in reverse order, -references are released. An object with `lifetime: "referenced"` (the default) ends when its last reference goes, -after the type's `linger` if it has one. A `lifetime: "host"` object (only for types without a required input) lives -as long as the host. A running, finishing or blocked job counts as a reference (chapter XI). - -## `ref.state` - -Every reference reports what is going on as a tagged value; listeners get each new one: - -```ts -type RefState = - | { kind: "loading" } // first hold, reconnecting, restarting - | { kind: "live" } - | { kind: "failed"; cause: FailureCause; retryAt: number | undefined } // undefined: no automatic retry - | { kind: "released" }; -``` - -| Situation | Calls | Values | -| --- | --- | --- | -| link drop | park until reconnected | readable, `loading` | -| object loading or restarting | park until live | readable, `loading` | -| rollout: provider ahead or installing | park until installed | readable | -| object `failed` | reject at once with the cause | `failed`, with cause and `retryAt` | - -Values stay readable through all of this: the panel keeps showing the last notes. Parked calls share one deadline (60 -s by default, `parkTimeout` per `get`), then fail `unavailable`. Otherwise a call ends only through its caller's -signal; there is no per-call deadline. A call that was running on an instance that ends fails with an error and is -not resent: Chord cannot know whether it took effect. - -**Errors look like disconnects.** A consumer handles a failed call exactly as a dropped link: show the error, let the -user retry. Work that must not run twice carries its own idempotency key as data (Durable's `requestId`, a job's -`key`). - -:::steps | The worker restarts while a tab watches the notes -+++ Watching -The tab holds `Notes { S1 }`. `ref.state` is `live`; `notes.notes.value` has 40 notes. -+++ The worker goes away -The supervisor stops worker S1 (or it crashes). The server's IPC link to it drops. The server tells the tab that its -hold is gone; the tab's reference goes `loading`. The 40 notes stay readable. -+++ A click meanwhile -Ann's colleague clicks **Add**. The call **parks** on the reference; nothing is sent yet. The deadline is 60 s. -+++ Chord resolves again -Holds on S1 still exist, so Chord calls `route(Session).resolve({ id: "S1" })` again, with the per-address backoff. -The app forks a new worker: a new **incarnation** of `Session({ id: "S1" })`. -+++ Re-hold -The server re-holds every hold on S1 over the new link. The new worker activates `NoteStore { S1 }` (its notes load -from storage) and the views; it sends fresh snapshots. The tab's reference goes `live`. -+++ The parked call runs -The parked `add` is sent now and succeeds. The panel shows 41 notes. Nobody wrote recovery code. -::: - -## Failure and retry - -Any broken part fails the **whole** object: a half-working object could break rules that rely on its documents -changing together. Chord tags the cause by where it failed: - -```ts -type FailureCause = - | { kind: "transient"; error: ChordError } // an open threw, a dependency was unreachable: automatic backoff - | { kind: "broken"; error: ChordError }; // factory, derive, hydration or migrate threw: retried only by a trigger -``` - -**Transient** causes get a backoff per address: 1 s, doubling to 60 s; concurrent gets share one attempt; success -resets it. **Broken** causes would fail the same way again (a code bug, invalid data), so they are retried only by a -**trigger**: a plugin reload, `ref.retry()` (a Retry button), or a readable the factory declared with -`self.retryOn(...readables)`. Only an opener may declare its own failure broken, with `throw broken(error)`, for -example for invalid configuration. A failed reference stays held; on success it goes `loading`, then `live`. - -```ts -host.serve(EnvConnection, ({ env }, self) => { - const config = self.get(EnvConfig, {}); // a facet or registry value, outlives the attempt - self.retryOn(config.envs); // a config change retries a broken open - const ssh = self.open(async ({ signal }) => { - const c = config.envs.value[env]; - if (!c) throw broken(new Error(`unknown environment ${env}`)); - return connect(c, signal); - }, (s) => s.close()); - … -}); -``` - -`retryOn` sources must live independently of the attempt (facet documents, registry values); a member of one of the -object's own references is a type error, since it goes away with the attempt. - -## Waiting for dependencies - -An object whose references are not live yet stays `loading` and goes live the moment they are, bounded by -`readyTimeout` (60 s). The decision for the failure case (R18) is what makes recovery automatic: - -:::sequence animate | Dependency recovery. The view fails transient with the store's cause, keeps its reference, and is retried the moment the store goes live. -actor tab "tab" -actor view "Notes { S1 }" blue -actor store "NoteStore { S1 }" accent -tab -> view: hold -view -> store: self.get (activation) -store -> store: open fails: disk not mounted -store --> view: failed (transient) -view --> tab: failed: transient, cause inside -note view..store: the view keeps its reference; its state is the retry trigger -store -> store: backoff retry: open succeeds -store --> view: live -view -> view: retried at once (backoff cancelled) -view --> tab: live -::: - -A dependency that fails, or does not become live in time, fails the dependent **transient** with the dependency's -cause inside, never broken: the cause is the dependency's. The dependent keeps its references while failed, and Chord -uses their state as a retry trigger. Factory authors write `self.get` and know none of this. After an object is live, -a dependency leaving `live` does not change it: its calls that use the dependency get the dependency's error, or park -while it loads. - -**Cycles are rejected** (Q5). If A's activation gets B and B's gets A, neither would ever lose its last reference. -Every object instance has a uuidv7 instance id; on one host Chord follows the waiting edges of activating objects, and -across hosts a hold made during an activation carries the chain of instance ids. A cycle fails the activation -`broken`, naming it ("A → B → A"), and keeps no references. - -## No optional references - -`get` has no `optional` flag. When a dependency's plugin may be missing, turn the dependency around: the dependent -owns a registry and the other plugin contributes to it. Radius uses forwards from the ports plugin, but ports is -optional, so radius owns `radius.forwards` and ports contributes its forwards; without ports the table is empty and -radius works (chapter XIII). - -## `ctx` - -Every call gets a `ctx` as its last argument: - -| Field | Meaning | -| --- | --- | -| `signal` | fires when the caller cancelled or the object ended; pass it to everything you wait on | -| `principal` | the caller (remote types only) | -| `incarnation` | the calling host's incarnation, for calls that arrived over a link (chapter X) | -| `get` | references released when the call ends | -| `telemetry`, `trace(name, attributes, fn)` | the current span; a child span with a child `ctx` (chapter XVII) | -| `jobId`, `progress` | in a job's `run` (chapter XI) | - -```ts -exportText: async ({}, ctx) => { - const catalog = await ctx.get(Catalog, {}); // released when the call ends - const title = catalog.sessions.value[session]?.title ?? session; - const res = await fetch(renderUrl, { method: "POST", body: render(title, list.value), signal: ctx.signal }); - return await res.text(); -}, -``` - -There are no code versions and no request ids in `ctx`. Durable's `Context` is the same module (`chord/context`): -`ctx` extends it, so passing `ctx` into pi-durable needs no conversion. Code outside objects gets a `ctx` from -`host.root(fn, { principal?, signal? })`; inside Durable tools it uses `chord(fn, context)` (chapter XIV). - -:::note | Built so far -References, `ctx` and the lifecycle are layers L4 and L5. `ChordError` and `FailureCause` are specified in the L1 -draft (`layer-1.md`), with codes `invalid`, `contract_mismatch`, `forbidden`, `gone`, `not_ready`, `released`, -`too_large`, `unavailable` and `aborted`, and a finer `reason` such as `invalid/args`. -::: diff --git a/packages/chord/docs/next/explainer/src/07-revocation.md b/packages/chord/docs/next/explainer/src/07-revocation.md deleted file mode 100644 index 1cacf921b..000000000 --- a/packages/chord/docs/next/explainer/src/07-revocation.md +++ /dev/null @@ -1,113 +0,0 @@ - -# Taking Access Away {#revocation lede="Revocation: how removing a guest ends their objects on every host at once, and why reconnecting does not bring access back."} - -The owner removes Ann from S1. Her tab still has a WebSocket open and a `Notes { S1 }` view on worker S1. Both must end -now, on the server and on the worker, and her tab's automatic reconnect must not restore anything. Chord provides the -mechanism; the app decides who is blocked and remembers it. - -## Labels, not predicates - -The app declares, once, which **revocation labels** each principal carries. Revoking means revoking one label; -everything carrying it is hit. - -```ts -// studio/src/principal.ts -export type Revocation = { grant: string } | { device: string } | { session: string }; - -export const revokedBy = (p: P): Revocation[] => - p.kind === "guest" ? [{ grant: p.grant }, { session: p.session }] - : p.kind === "owner" && p.access === "device" && p.device ? [{ device: p.device }] - : []; - -const server = createHost({ address: Server(), principal: { kind: "server" }, revokedBy }); -// every worker is created with the same revokedBy -``` - -| Principal | Labels | -| --- | --- | -| guest Ann (grant g1) in S1 | `{ grant: "g1" }`, `{ session: "S1" }` | -| guest Bob (grant g2) in S1 | `{ grant: "g2" }`, `{ session: "S1" }` | -| owner on the paired phone | `{ device: "phone" }` | -| owner, local | none | - -| Call | Hits | -| --- | --- | -| `server.revoke({ grant: "g1" })`: remove Ann | Ann | -| `server.revoke({ session: "S1" })`: archive S1 | Ann and Bob | -| `server.revoke({ device: "phone" })`: sign out the phone | the phone | - -A label is plain data because revocation is an event that must travel from the server to the workers; a predicate -function could not be sent. - -## What `revoke` does - -`host.revoke(r)` ends every object and closes every link whose principal `p` has `r` in `revokedBy(p)` (compared by -canonical JSON; never the host's own principal), and passes `r` to its children, which apply their own `revokedBy`. - -:::sequence animate | Removing guest Ann. The label travels down; her objects end on every host; her reconnect is refused by the app's own record. -actor owner "owner tab" -actor server "server" accent -actor worker "worker S1" blue -actor ann "Ann's tab" "guest g1" muted -owner -> server: Sharing.remove({ grant: "g1" }) -server -> server: app: mark g1 blocked (document) -server => server: revoke({ grant: "g1" }) -server -> ann: close link -server => worker: revoke { grant: "g1" } -worker -> worker: end Notes { S1 } as g1 -ann -> server: reconnect -server -> server: authenticate: g1 blocked -server --> ann: refused (4401) -::: - -On worker S1, for `revoke({ grant: "g1" })`: - -| Object | Principal | Hit | -| --- | --- | --- | -| `Notes { S1 }` | owner, local | no | -| `Notes { S1 }` | guest g1 | **yes**: ends | -| `Notes { S1 }` | guest g2 | no | -| `NoteStore { S1 }` | none (local) | no | - -Ending Ann's view releases its reference to the store, which keeps serving the others. - -## Chord persists nothing here - -`revoke` cuts live access only. Lasting denial lives in the app's data (the sharing plugin's guest records), which -`authenticate` checks on every new connection. After a server restart nothing live exists to revoke; every client -authenticates again and blocked grants are refused there. - -**No re-stamping.** A changed grant (Ann moves from read to write) is a revocation of the old grant. Her link closes, -her tab reconnects and authenticates with the new grant revision, which gives a new principal key, and her references -re-hold by themselves, now as an editor. Reconnecting never restores revoked authority. - -**Revocation beats jobs** (R11). Normally a running job keeps its object alive after the last reference is released -(a closed tab does not abort an import). Revoking a principal ends its objects at once regardless; their jobs get the -abort signal and the normal close timeout. - -## Work in shared stores - -Revocation ends objects whose principal carries the label. A shared store has no principal, so work it runs *on -behalf of* someone (a question waiting for Ann's answer, a shell command a guest started) is not ended by `revoke`. -Such a store keeps the grant as data with the work and reconciles from the grant owner's live state with a routine -(chapter X): - -```ts -// in a shared store that runs work for guests -const sharing = self.get(Grants, {}); // the sharing plugin's live grants -self.routines.on(sharing.live, async (live, ctx) => { - for (const run of shell.running.value) - if (run.args.grant && !live[run.args.grant]) shell.abort(run.id); // idempotent -}); -``` - -There is no revocation hook: a hook can be missed (a crash between revoke and hook), a routine over current state -cannot. The window between the revocation and the routine's run is accepted (R10); a step started in it still runs. - -## Who does what - -| Chord | The app | -| --- | --- | -| stamps principals on links | what a grant is; share links, cookies, `authenticate` | -| keeps one object per principal key | defines `Revocation` and `revokedBy` | -| on `revoke(r)`: ends matching objects and links, passes `r` down | decides when to revoke; stores who is blocked | diff --git a/packages/chord/docs/next/explainer/src/08-derives.md b/packages/chord/docs/next/explainer/src/08-derives.md deleted file mode 100644 index 720542ef5..000000000 --- a/packages/chord/docs/next/explainer/src/08-derives.md +++ /dev/null @@ -1,114 +0,0 @@ - -# Computed Values {#derives lede="Derives, the one footgun that makes them expensive, mapShared, and presence: who is looking."} - -The reader's notes without authors, a note count, the number of people viewing: values computed from other values. -Storing copies would go stale; recomputing them on every read would be wasteful and could not be watched. A -**derive** is computed once per change and sent to viewers as a difference. - -## `self.derive` - -```ts -const count = self.derive("count", [list], ([l]) => l.items.length); -const summary = self.derive("summary", [list, store.pinned], ([l, pinned], previous) => ({ - total: l.items.length, - pinned: pinned.length, -})); -``` - -`self.derive(name, sources, (values, previous) => value)`. Sources can be documents, readables, other derives and -state members of references. The value is recomputed when a source changes and diffed into operations for the wire. -Derived values are adopted before ordinary observers run and share their source commit's wire batch, so a viewer -never sees a note list and a count from different commits. - -`previous` is the derive's last value, kept across factory re-runs (reloads) by name. It is a cache, empty after a -host teardown, so a derive must be correct without it. A derive that throws fails its whole owner `broken`. - -## How the difference is found - -Chord compares the new value with the previous one using Delta's `diffRevisions`. Parts that are the **same objects** -as last time cost one `===` each. Parts that are new objects must be compared by content. - -**The footgun.** A derive that builds new objects for unchanged data makes every commit cost the size of the whole -value, per view. The reader list from chapter IV, written naively: - -```ts -self.derive("public", [list], ([l]) => ({ items: l.items.map((n) => ({ id: n.id, text: n.text, at: n.at })) })); -// one note added to 2,000: 2,001 new objects, every one compared by content -``` - -**`mapShared(f)`** keeps one result per source element: `f` runs only for new or changed elements, unchanged ones -return their previous result object. - -```ts -import { mapShared } from "@earendil-works/chord/delta"; - -const stripAuthor = mapShared((n: Note): PublicNote => ({ id: n.id, text: n.text, at: n.at })); -self.derive("public", [list], ([l]) => ({ items: l.items.map(stripAuthor) })); -// one note added: one new object, 2,000 identical references; the diff emits one insert -``` - -:::review | Review: mapShared is planned, not available -Design §12 introduces `mapShared` as coming "from `@earendil-works/chord/delta`", but nothing in `packages/chord/src` -defines it yet, and Delta does not yet treat bytes as atomic leaves in `diffRevisions`. Design §28 lists both as the -step "Delta in pi", before the layered rebuild. Until then, keep a `Map` from source element to result yourself, which -is what `mapShared` will do. -::: - -## Cookbook - -| Case | How | -| --- | --- | -| select a field, look up by key, a small aggregate | compute it; the value is small | -| filter or map a list | `mapShared` over the elements | -| redaction | preferably separate documents per visibility; otherwise `mapShared`, computed once in the shared store, never per view | -| join two sources | `mapShared` plus the looked-up source's revision | -| a window over a large list | take a section of the source elements without rebuilding them; cache per-element measurements by identity; keep the window's start stable so changes arrive as appends | -| a large view combined from several sources | re-path the sources' operations, as Durable's `ConversationView` does; application code never interprets ops | - -**Container width** (R24): structural sharing is O(change) along a path, not across a container's width. Changing one -item of a flat array copies the array's slots, and `mapShared` visits every element once (one identity comparison -each). Large collections are keyed documents or bounded windows. A benchmark of long arrays is to set the documented -limit. - -## Readables that are not documents - -Three small readables round out the toolkit: - -- `constant(value)`: never changes. Start data uses it (chapter IX). -- `self.writable(initial)`: a host-local readable of any in-process value (functions included), compared by identity, - with `set`. It never goes on the wire; it exists for registry contributions that change (chapter XIII). -- Source readables from elsewhere: a Durable view or document watch is already a Chord readable, because Durable and - Chord v2 share the readable type (`chord/src/state.ts`), so it can feed a derive or be exposed as state directly. - -:::review | Review: exposing a writable (Q14, open) -A `writable` may hold functions; nothing yet stops a factory from returning one as a state member, which would reach -the wire encoder. Review round 3 proposes that only members with a JSON schema in the contract are exposable, checked -when the object is served. Undecided. -::: - -## Presence: who is looking - -`host.references(Type)` is a readable of who holds references to objects of a type on this host: input, principal and -client. Client holds carry an opaque origin id, so two tabs are two clients. - -```ts -host.serve(NoteStore, ({ session }, self) => { - const viewers = self.derive("viewers", [host.references(Notes)], ([refs]) => - refs.filter((r) => r.input.session === session).length); - return { …, viewers }; -}); -``` - -A link drop is a departure: accurate and simple. Smoothing (a phone switching networks makes the count flicker) is the -UI's job, for example a debounced count. Counts per person are derived by the app from principals and origins. There -is no consumer-side linger: a closed tab closes its link and keeps nothing. The provider's `linger` keeps a -just-closed object warm, and that also keeps the worker alive through `host.activity`. - -**"Leaving" is never derived from an unmount.** A UI reload unmounts and remounts components; a component unmounted by -a reload has not left. Leaving is explicit navigation, or presence (a dropped reference). This replaces p2's presence, -viewer and login-abandon leases with one mechanism. - -:::note | Built so far -Readables and derives are layer L2 (a readable is today's `replicatedState`, derives reuse today's derived cells over -it). Not started in the rebuild. -::: diff --git a/packages/chord/docs/next/explainer/src/09-natives.md b/packages/chord/docs/next/explainer/src/09-natives.md deleted file mode 100644 index ed7dacbbf..000000000 --- a/packages/chord/docs/next/explainer/src/09-natives.md +++ /dev/null @@ -1,160 +0,0 @@ - -# Things the OS Gives You {#natives lede="Start data, host-lifetime objects and natives: how a plugin opens a file watcher or a socket so that it is closed exactly once, also across reloads."} - -Studio's notes get an inbox: a text file dropped into the session's `inbox/` directory becomes a note. That needs a -file watcher, a **native**: an OS-level value (a file handle, a socket, a process, a database connection) that cannot -be JSON, cannot be copied, must be opened, and must be closed exactly once. It also needs the session's data -directory, which the plugin does not know. This chapter covers both. - -## Start data is an object - -A worker knows things before any plugin loads: its session id, the data directory, init data from the server's start -message. There is no special mechanism for them. Start data is an ordinary `local` object with `lifetime: "host"` and -constant state, served by base code's core facet: - -```ts -// studio/src/core/contract.ts -export const Init = obj({ id: str(), dataDir: str(), cwd: str(), env: opt(str()) }); -export const SessionInfo = local("studio.session-info", { - version: 1, - input: obj({}), - members: { init: Init }, -}); - -// studio/src/core/worker.ts -export const coreWorkerFacet = (init: Static) => defineFacet((host) => { - host.serve(SessionInfo, () => ({ init: constant(init) }), { lifetime: "host" }); - // … the Harness and other core objects (chapter XIV) -}); -``` - -Any plugin on the worker declares a reference and reads it **after activation**: in an opener, a derive, a routine or -a method, never synchronously in the factory. - -```ts -const info = self.get(SessionInfo, {}); -// later: info.init.value.id, info.init.value.dataDir -``` - -Paths are derived by the plugin from generic start data (`join(dataDir, id, "notes-inbox")`), never provided per -plugin. Platform bindings work the same way: on a Durable Object, a core object's `self.open` returns the binding -(`state`, `env`) and its close does nothing. - -## `self.open` - -```ts -const watcher = self.open( - async ({ signal }) => watchDir(path, { signal }), // open: during activation - (w, reason) => w.close(), // close: when the object ends or restarts - { closeTimeout: 10_000 }, // optional; host default 30 s -); -watcher.value; // the watcher; reading it before activation throws not_ready -``` - -`self.open` returns a handle with `.value` (type `Native`), not a proxy. The rules: - -- **Natives live only in objects.** Not at module level, not in the facet. A module-level watcher would survive a - reload with nobody to close it. -- **Order:** natives open one after another in declaration order (a later opener may read an earlier handle's - `.value`) and close in reverse. The object goes live after all opens succeeded. -- **The value never changes during an attempt.** No replacement, no versions, no stale-handle logic. To change a live - native, the object restarts. -- **A failed open** makes the object `failed`: transient with backoff by default, broken if the opener says so - (`throw broken(error)`). A partly failed activation closes the natives already open, with reason `end`; the next - attempt opens all of them anew. -- **`close` gets a reason:** `{ kind: "reload" }` (the plugin reloads and this native object restarts), `{ kind: - "stop" }` (the host shuts down) or `{ kind: "end" }` (the last reference ended, or the type was removed). Each - `close` is bounded by `closeTimeout`, then reported as a possible leak while the rest continues. A throwing `close` - does not stop the others; `host.close()` collects the errors, background ends report to `onError`. -- **Behaviour around a native goes into routines** (chapter X). Listeners on natives outside routines are forbidden. - -An object with at least one `self.open` is a **native object**; one without is a **pure object**. The difference -decides what a reload does (chapter XV): pure objects are swapped in place, native objects restart. - -## The inbox - -The inbox must run as long as the worker, whether anyone looks at the notes or not. So it is a `lifetime: "host"` -object. The facet cannot `self.get` a type it serves, and the inbox has no meaningful input, so it is an object -without input that reads its session from the start data: - -```ts -// notes/contract.ts -export const NoteInbox = local("notes.inbox", { version: 1, input: obj({}), members: {} }); - -// notes/worker.ts -host.serve(NoteInbox, (_input, self) => { - const info = self.get(SessionInfo, {}); - const watcher = self.open(async ({ signal }) => { - const { id, dataDir } = info.init.value; // read in the opener: after activation - return watchDir(join(dataDir, id, "notes-inbox"), { signal }); - }, (w) => w.close()); - - self.routines.loop("import", async (ctx) => { // chapter X - const store = await ctx.get(NoteStore, { session: info.init.value.id }); - for await (const file of watcher.value.files(ctx.signal)) { - await store.add({ text: await readFile(file, "utf8"), by: "inbox" }, ctx); - await rm(file); - } - }); - return {}; -}, { lifetime: "host" }); -``` - -Note `ctx.get(NoteStore, …)` in the routine rather than `self.get` in the factory. The factory runs before the -session id is known (start data is read after activation), so a host-lifetime object without input gets -session-scoped objects with `ctx.get` in a routine or method (R22). A `get` whose input is computed after activation is -to be added only if a real case needs it. - -## Natives with work in flight - -A restart is sequential (R16): close the old native, then open the new one. When the old one still has work running, -use one of two patterns: - -- **Drain in the background.** `close` detaches the old resource and returns at once; it disposes of it when its work - finishes, or after `closeTimeout` at most. Chord opens the new one immediately. Code mode's sandbox does this: a - script running during a reload finishes on the old sandbox (for up to 30 s, Q7; a longer one is aborted and its turn - fails), new scripts start on the new one. -- **Swap through a registry.** The resource is a registry contribution (chapter XIII). Readers always look up the - current one at use; a reader that must wait for one uses a cancellable "wait until present" with the park deadline. - -Chord aborts only the object's own calls and jobs on close. Other work that uses the native (Durable tool calls on a -sandbox) is the owner's business: `close` may drain instead of tearing down. What an old native does after its owner -restarts is also the owner's `close`: a model runtime keeps working until collected, a closed SSH connection fails its -in-flight `exec`. - -## Natives that follow a relocation - -A session can move to another directory. The inbox watcher must follow. Restarting the object on every move would be -heavy-handed; instead the native is a stable manager opened once, and a routine on the session location switches its -internal resource and bumps a generation, so late events from the old location are dropped: - -```ts -const watcher = self.open(async ({ signal }) => new SwitchableWatcher({ signal }), (w) => w.close()); -self.routines.on(location.cwd, (cwd) => watcher.value.switchTo(join(cwd, ".notes-inbox"))); -``` - -## Call-scoped natives - -A native that lives exactly as long as one call (a temporary SSH session for a directory listing, a file handle for a -download) is opened and closed inside the method with `try/finally`. There is no `ctx.open`: - -```ts -list: async ({ path }, ctx) => { - const session = await ssh.open(config, ctx.signal); - try { return await session.readDir(path, ctx.signal); } - finally { session.close(); } -}, -``` - -## Core vs. plugin - -`SessionInfo` comes from base code, `NoteInbox` from a plugin; both use the same primitives. Where code lives decides -which it is. **Base code** (the app's `src/`, imported statically by the start code under a fixed name such as -`@studio/core`) is always there, not switchable, and a change restarts the hosts. **A plugin** (its own directory, -found by listing, named by its `package.json`) is switchable and reloads. Core is what the app always needs and -everything builds on: the harness, the session list, access, the session location. Optional features are plugins; -code mode is a plugin. - -:::note | Built so far -Natives and host-lifetime objects are layer L4. `constant` is decided (R23), not built. -::: diff --git a/packages/chord/docs/next/explainer/src/10-routines.md b/packages/chord/docs/next/explainer/src/10-routines.md deleted file mode 100644 index 4191f21d0..000000000 --- a/packages/chord/docs/next/explainer/src/10-routines.md +++ /dev/null @@ -1,145 +0,0 @@ - -# Standing Behaviour {#routines lede="Routines, reconciling instead of notifying, and how two owners keep a rule between them without a shared transaction."} - -Notes older than 30 days should go every hour. Inbox files should be imported as they arrive. Notes of an archived -session should become read-only. Each of these is behaviour that stands, not a call someone makes. Written as bare -`setInterval`s and event listeners, it would keep running old code after a reload and touch state from nowhere. Chord -has one declared form for it: the **routine**. - -## Four kinds - -```ts -self.routines.every("cleanup", HOUR, (ctx) => dropOlderThan(30 * DAY, ctx)); // due time survives reloads -self.routines.on(settings, async (value, ctx) => applyLimit(value, ctx)); // a readable or event source -self.routines.loop("import", async (ctx) => { // runs until the object ends - while (!ctx.signal.aborted) await importNext(ctx); -}); -self.routines.custom("ticker", ({ signal, run }) => { // anything else - const timer = setInterval(() => run(tick), 1000); - return () => clearInterval(timer); // cleanup undoes the setup -}); -``` - -Routines start when the object goes live, stop when it ends, and restart with the new code on every reload of their -plugin. Each run gets its own `ctx`; errors are caught and reported, never uncaught. Routines are named (`on` -optionally by its source) so Chord recognizes them across reloads; that is how `every` keeps its due time. - -## Serialized per routine - -The new instance of a routine starts only after the old handler in flight has settled; only that routine waits, for at -most the routine handover timeout (10 s), after which it starts anyway and the stuck handler is reported. - -The reason, as a trace: a supervisor routine reconciles pending worker restarts. The old handler is midway through -`restart(W1)` when the plugin reloads. Without serialization the new routine starts at once, sees W1 still pending, and -restarts it a second time. - -## Reconcile, don't track - -A routine **reconciles from the current value**: it makes the world match what it sees now. It writes no "in -progress" flags, and it starts a job for long work (chapter XI). A handler that only cares about the latest value -compares with the current value itself; there is no `overlap` option. Subscribing to the object's own documents is -gapless: every commit is seen. - -No bare timers or callbacks that touch state outside routines, jobs or calls. That is what lets Chord stop all of an -object's behaviour when it ends or reloads. - -## Rules between two owners - -Every thing has exactly one owner, the smallest object it belongs to. Some rules span two: "notes of an archived -session are read-only" involves the catalog (which knows archiving) and `NoteStore` (which holds notes). Two tempting -designs are wrong: - -- **A cross-owner transaction** (archive and lock the notes atomically) would make the catalog know every plugin and - couple their storage. -- **A notification** (`catalog` calls `notes.archive(session)`) can be lost: the worker is not running, the link drops, - the process crashes between the archive and the call. - -The design's rule: **the owner of the dependent data enforces the rule** by reconciling from the other owner's state -with an idempotent routine. - -```ts -// NoteStore: an archived session's notes are read-only -const flags = self.state(NoteFlags); // { archived: boolean }, NoteStore's own document -const catalog = self.get(Catalog, {}); // the server's session list: another owner -self.routines.on(catalog.sessions, async (sessions, ctx) => { - const archived = sessions[session]?.archived ?? false; - if (flags.value.archived !== archived) await flags.change((d) => { d.archived = archived; }, ctx); -}); -// add and remove refuse while flags.value.archived -``` - -After a crash the routine runs again with the current value. The catalog does not know the notes plugin exists. The -same pattern closes a forward of an archived session in the ports plugin, aborts a job whose grant was revoked -(chapter VII), and keeps any invariant between plugins. - -## Reading without starting a worker - -The session list in the sidebar shows each session's note count. Getting `NoteStore { S1 }` for every session would -start every worker. The answer is plain publishing between two objects of one plugin, no Chord mechanism: the worker -side reports a summary to a server-side object by a method call from a routine, and clients read the server object. - -```ts -// notes/contract.ts -export const NoteCounts = remote("notes.counts", { - version: 1, input: obj({}), placement: () => Server(), - variants: { - owner: { state: { counts: map(int()) } }, - worker: { methods: { report: method(obj({ session: str(), count: int() })) } }, - }, -}); - -// notes/server.ts -const Counts = defineState("notes.counts", { - schema: obj({ bySession: map(obj({ count: int(), incarnation: str() })) }), - version: 1, initial: () => ({ bySession: {} }), persist: true, // survives a server restart - checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64, -}); - -host.serve(NoteCountStore, (_input, self) => { // local, lifetime "host" - const counts = self.state(Counts); - return { - counts: self.derive("counts", [counts], ([c]) => mapValues(c.bySession, (e) => e.count)), - report: ({ session, count, incarnation }, ctx) => - counts.change((d) => { - const last = d.bySession[session]; - if (last && last.incarnation > incarnation) return; // uuidv7: an older worker, ignore - d.bySession[session] = { count, incarnation }; - }, ctx), - }; -}, { lifetime: "host" }); - -host.serve(NoteCounts, (_input, principal, self) => { - const store = self.get(NoteCountStore, {}); - if (principal.kind === "worker") return { - variant: "worker", - report: ({ session, count }, ctx) => { - if (session !== principal.session) throw forbidden(); // a worker reports only itself - return store.report({ session, count, incarnation: ctx.incarnation }, ctx); - }, - }; - return { variant: "owner", counts: store.counts }; -}); - -// notes/worker.ts, inside NoteStore -const counts = self.get(NoteCounts, {}); // a reference to the server -self.routines.on(count, (n, ctx) => inVariant(counts, "worker") && counts.report({ session, count: n }, ctx)); -``` - -The server object keeps the latest value per session (in a document if it must survive a server restart) and -**ignores reports from an older worker incarnation** (incarnations are uuidv7, so they order by start time). A worker that was replaced may still deliver a late report; the -fence is the caller's incarnation, which Chord stamps on every call from the link handshake (R20): `ctx.incarnation`, -next to `ctx.principal`, not part of the principal or its key, never sent as data, so it cannot be forged. Session -status, subagent lists, unread marks and artifact lists in p2 all follow this pattern. The store is local and has no -caller incarnation of its own, so the view passes it on as data. - -:::review | Review: `ctx.incarnation` across a relay (Q13, open) -A worker's call to the server arrives directly, so `ctx.incarnation` is the worker's. But a call relayed through the -server (a tab's call to a worker) carries the server's incarnation on the worker's side, because the worker's -handshake is with the server. Review round 3 proposes that relays stamp the origin host's incarnation, as they stamp -the principal. Undecided; the pattern above only needs the direct case. -::: - -:::note | Built so far -Routines are layer L8. The design treats `every` as one of four forms over one runner; review round 3 proposed cutting -to `on` and `loop` only (C4), not decided. -::: diff --git a/packages/chord/docs/next/explainer/src/11-jobs.md b/packages/chord/docs/next/explainer/src/11-jobs.md deleted file mode 100644 index e506c05df..000000000 --- a/packages/chord/docs/next/explainer/src/11-jobs.md +++ /dev/null @@ -1,171 +0,0 @@ - -# Work That Outlives the Call {#jobs lede="Jobs: owned work started by a method that keeps running after its caller leaves, with cancel, dedupe and a recorded outcome."} - -Importing a 50 MB text file into notes takes a minute. The tab that started it may close; the import must go on, be -cancellable, not run twice when the tab retries after a link blip, and its result must be recorded. A method cannot do -this (it dies with its caller), a routine is standing behaviour rather than a request, and a Durable task is for work -that must resume after a crash. Chord's form for it is the **job**, built in pi-durable's sandwich: **intent** (a -transaction), **effect** (anything), **outcome** (a transaction in the current code). - -## Defining and running a job - -```ts -// notes/worker.ts -const ImportJob = defineJob("notes.import", { - version: 1, - args: obj({ path: str() }), - progress: obj({ lines: int() }), - result: obj({ added: int() }), - concurrency: 1, // one import at a time per NoteStore -}); - -const ImportResults = defineState("notes.import-result", { - schema: obj({ outcome: union([obj({ added: int() }), obj({ error: str() })]) }), - version: 1, initial: () => ({ outcome: { added: 0 } }), keyed: true, - checkpointWhen: () => true, -}); - -host.serve(NoteStore, ({ session }, self) => { - const list = self.state(NoteList); - const results = self.state(ImportResults); - const imports = self.job(ImportJob, { - run: async ({ path }, ctx) => { // 2. effect - let lines = 0; - for await (const line of readLines(path, ctx.signal)) { - await addNote({ id: `${ctx.jobId}:${lines}`, text: line, by: "import" }, ctx); // keyed by job id - ctx.progress({ lines: ++lines }); - } - return { added: lines }; - }, - finish: async (outcome, tx) => { // 3. outcome, current code - (await tx.draft(results, outcome.id)).outcome = - outcome.kind === "done" ? outcome.result : { error: outcome.error }; - }, - }); - return { - …, - imports: imports.running, // live list of runs - startImport: ({ path, requestId }, ctx) => - imports.start({ path }, ctx, { key: requestId }), // 1. intent (refined below) - abortImport: ({ id }) => imports.abort(id), - }; -}); -``` - -`defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })`; `progress` and the outcome -are typed by the schemas. - -:::sequence animate | The job sandwich. The intent commits before anything runs; the effect runs outside any transaction; the outcome commits in the current code. -actor tab "tab" -actor store "NoteStore { S1 }" accent -actor run "run" "effect" blue -tab -> store: startImport({ path, requestId }) -store => store: 1. intent: busy check, dedupe by key, commit -store --> tab: { id } -store -> run: start after the commit -run -> run: read lines, add notes (keyed by jobId) -run --> store: progress { lines } (throttled) -note tab..store: the tab may close here: the job counts as a reference -run --> store: settles: { kind: "done", result } -store => store: 3. record outcome, finish(outcome, tx), ack -::: - -## The three steps - -**1. Intent.** `job.start(args, ctx, { key? })` is shorthand for `tx.start(job, args, { key? })` inside a -transaction, which may contain other writes. The busy check (`concurrency`) and deduplication (the same key returns the -existing run) are atomic and in the same commit; `run` starts only after the commit. Run ids are uuidv7. - -**2. Effect.** `run(args, ctx)` may do anything: processes, network, other objects. It writes no document itself; it -returns the outcome. External writes carry a dedupe key derived from `ctx.jobId` (above: note ids), so an interrupted -attempt cannot apply them twice. On a reload `run` keeps its old code. - -**3. Outcome.** When `run` settles, Chord records the outcome in its own record, then calls the **current** code's -`finish(outcome, tx)` and acknowledges the run in the same commit. The outcome is tagged: - -```ts -type Outcome = { id: string } & ({ kind: "done"; result: R } | { kind: "failed"; error: "threw" | "aborted" | "lost" }); -``` - -A throwing `finish` changes nothing and is retried with a backoff (1 s to 60 s), and after the next reload. `finish` -is optional and follows Durable's transaction rule. - -## Dedupe ends with the run - -Deduplication covers **unsettled runs only** (Q9). Once `finish` has committed, Chord drops the run's record and key -and releases its waiters' references; it keeps nothing beyond a run's lifetime. Dedupe after completion is the -owner's. Studio records which run a request started, in the same commit as the intent, and `finish` records the -result by run id (`ImportResults` above): - -```ts -const Requests = defineState("notes.import-request", { - schema: obj({ run: str() }), version: 1, initial: () => ({ run: "" }), keyed: true, persist: true, - checkpointWhen: () => true, -}); - -startImport: async ({ path, requestId }, ctx) => { - const prior = await requests.get(requestId); // read outside the transaction (R13) - if (prior) return { id: prior.run }; // started before, maybe finished long ago - return self.tx(async (tx) => { - const run = await tx.start(imports, { path }, { key: requestId }); // dedupes a concurrent retry - (await tx.draft(requests, requestId, { tags: [SessionTag(session)] })).run = run.id; - return { id: run.id }; - }, ctx); -}, -``` - -Two concurrent retries both miss `prior`, but the key makes `tx.start` return the same unsettled run. The client -watches `imports` for progress and reads the result by run id once it is gone from `running`. - -Why Chord does not keep completed runs itself: either 100,000 keyed imports would stay in memory forever, or a -retention limit would let a retried key rerun its effect. The owner knows which results matter. - -## Watching, waiting, aborting - -- **`running`** is a readable of `{ id, version, args, progress, phase }`. `phase` is `running`, `finishing` (outcome - recorded, `finish` not yet committed) or `blocked` (with the reason). `concurrency` counts only active runs. The - panel shows a progress bar from it. -- **Progress** is in memory, throttled and sent as deltas. For output, use `OutputBuffer` and `Progress` from - `@earendil-works/chord/output`: bounded windows, adaptive throttling, no write amplification. -- **Waiting:** `start` returns a run with `outcome(ctx)`. The caller's abort ends only its wait, never the run. -- **Abort is a request:** `job.abort(id)` fires the run's `ctx.signal`; the run decides what it returns (a result - that already arrived stays a success). A run that ignores the signal keeps its slot until it ends. `finish` runs in - Chord's own context. - -## Lifetime, persistence, revocation - -**Jobs keep their owner alive.** A running, finishing or blocked job counts as a reference to its object, which ends -only once all its jobs are settled. Starting an import and closing the tab does not abort it. Unsettled runs also -count as activity for the worker's keep-alive. - -**Revocation takes precedence** (R11). Revoking a principal ends its objects at once, whatever jobs remain; the jobs -get the abort signal and the normal close timeout. A job runs as its object's principal. - -**Persistence:** jobs are in memory by default and gone with the process. `persist: true` (only for stable -identities) keeps the record: a run live at a crash ends `lost` at the next activation and `finish` records it. -Nothing is rerun automatically. - -**Versions:** a run started under an older job version keeps its version; `finish` and `running` see it through pure -`accepts` adapters, and stored outcomes are never rewritten. A version bump without an adapter does not refuse the -reload: the old outcome is `blocked`, kept and reported until a later reload brings the adapter. - -**Native objects:** a job in a native object ends `aborted` when the object restarts on a reload (chapter XV). In a -pure object it keeps running on its old `run` code. - -## What is not a job - -| Work | Use | -| --- | --- | -| request and response that may die with its caller | a method | -| input into running work (a pasted code, an approval) | a flow (chapter XII) | -| work that must resume after a crash | a Durable task | -| standing behaviour | a routine | - -:::review | Review: jobs are the largest open simplification (#9, C3) -Decision #9 (deferred until login is ported): a login or a long relocation is a job whose state consumers want to -address and watch (`LoginFlow { id }`). Options: (a) jobs stay records inside their owner, watched through `running`; -(b) a job is an object of its own, addressed by its id, living until done. Separately, review round 3 proposed cut C3: -shrink jobs to a runner over documents (no job versions, `accepts`, `blocked`, `persist`) or move them out of the -kernel. Not decided. `chord-next-walkthrough.md` concept 17 still shows the older outcome shape `{ ok: true, result }`; -the design tags outcomes by `kind`. -::: diff --git a/packages/chord/docs/next/explainer/src/12-flows.md b/packages/chord/docs/next/explainer/src/12-flows.md deleted file mode 100644 index cf6a986c1..000000000 --- a/packages/chord/docs/next/explainer/src/12-flows.md +++ /dev/null @@ -1,141 +0,0 @@ - -# Asking a Person {#flows lede="Flows and the Questions service: how running work waits for an answer from someone who may be on any tab, may reconnect, and may answer twice."} - -A provider login asks for a pasted code, then an account choice. A dangerous tool call waits for the owner's -approval. The agent asks everyone present a question. In each case running work waits for a person, who may be on any -tab, may lose the connection halfway, may answer late or twice. Two patterns cover this: a **flow** kept as its -owner's state, and the **Questions** service, a shared store for questions. - -## A flow is state, not a call - -The tempting design is a long-held server call, `step()`, that blocks until the next step. A link drop fails it, and -the consumer cannot tell whether it missed a step. Chord's answer is that the owner keeps the flow as **state**, which -re-holds by itself after any drop: - -```ts -// models/contract.ts (sketch from the design) -start: method(obj({ provider: str() }), obj({ id: str() })) // uuidv7 -answer: method(obj({ id: str(), step: int(), value: str() }), Answered) // ok | "step no longer active" -cancel: method(obj({ id: str() })) - -// LoginFlow { id }: state -type FlowState = - | { kind: "running"; step?: { id: number; prompt: LoginPrompt }; notices: Notice[] } - | { kind: "done" | "failed" | "cancelled"; notices: Notice[] }; -``` - -The UI watches the state and renders the current step. A late or second answer to a step that is gone is refused. A -promise-based callback inside the work (pi-ai's `prompt` during a login) stays **inside** the owner: its job keeps the -resolver, and nothing crosses an object boundary. - -Imperative consumers (tests, scripts, a CLI, an agent tool) use a generic client helper over the same state, not a -second protocol: - -```ts -async function* steps(flow: LoginFlowRef, ctx: Ctx) { - let last = 0; - for (;;) { - const s = await until(flow.state, (v) => v.kind !== "running" || (v.step !== undefined && v.step.id > last), ctx); - if (s.kind !== "running") return s.kind; - last = s.step!.id; - yield s.step!; - } -} - -for await (const step of steps(await ctx.get(LoginFlow, { id }), ctx)) - await login.answer({ id, step: step.id, value: answerFor(step) }, ctx); -``` - -`last` keeps steps from being lost or repeated across re-holds. - -:::review | Review: where a flow lives (#11, deferred) -Decision #11, deferred until login is ported: (a) a keyed document of its owner, or (b) an object `LoginFlow { id }` -living until done; tied to #9 (jobs as objects). The `LoginFlow { id }` address above assumes (b). Also undecided is -how "leaving" ends a login (p2 cancels when the starting owner leaves everywhere); with presence it is a dropped -reference, never an unmount (chapter VIII). -::: - -## The Questions service - -The Questions service (`apps/p2/docs/questions.md`) is a p2 design, not a Chord feature, built from Chord's -primitives: a shared store with per-principal views. The asker gets `ask`; viewers get the open questions and -`answer`. - -```ts -export type Question = - | { kind: "text"; message: string; secret?: boolean; help?: string } - | { kind: "select"; message: string; options: readonly { id: string; label: string }[] } - | { kind: "confirm"; message: string; allow: string; deny: string }; - -type Policy = { mode: "first" } | { mode: "quorum"; count: number } | { mode: "all"; join?: "live" | "snapshot" }; - -ask(question, { - audience, // a readable or a list of principals - policy, - key?, // stable per asking flow: a re-ask reattaches to the pending question - deadline?, // then resolve with what is there; `missing` names who did not answer -}, ctx): Promise; -``` - -Its rules: - -- **The store is the truth.** Pending questions and their resolvers live in the store; `open` is a readable over it, - filtered per viewer. Nothing is a document, so a reload or migration cannot leave a ghost question. -- **One settlement point.** Register the waiter and the abort listener before publishing the question; an answer, an - abort or the deadline atomically takes the entry out before resolving. A late or second answer gets "no longer - active". Answers are validated against the stored question (a select id must be one of the options). Never await an - answer inside a transaction. -- **Policies over a live audience.** `first` resolves with the first answer; `quorum(n)` after n answers; `all` when - every member of the audience has answered. With `join: "live"` (the default), someone who joins becomes required and - someone who leaves drops out. -- **Authorization.** The asker is stamped by the host (a worker asks only in its own session's scope; secret questions - only from server askers). Each question stores who may answer it; a guest's send access does not imply approval - rights. Secret answers never appear in `open`, documents or logs. -- **Link drops and crashes.** A rejected `ask` is not a denial. Askers retry with the same `key` until their own signal - aborts; a pending `ask` counts as an active call for keep-alive. -- **Where it lives.** On the host where the asking work runs: session questions on the session's worker, login - questions on the server. UIs reach both through the usual relay. - -The store in Chord v2 terms: - -```ts -host.serve(QuestionStore, ({ scope }, self) => { - const pending = self.open(async () => new PendingQuestions(), (p) => p.rejectAll("store closed")); - const open = self.derive("open", [pending.value.readable], ([all]) => all); // data only, no resolvers - return { open, ask: (args, ctx) => pending.value.ask(args, ctx), answer: (args, ctx) => pending.value.answer(args, ctx) }; -}); - -host.serve(Questions, ({ scope }, principal, self) => { - const store = self.get(QuestionStore, { scope }); - if (isHost(principal)) return { variant: "asker", ask: (args, ctx) => store.ask({ ...args, asker: principal }, ctx) }; - if (!mayView(principal, scope)) throw forbidden(); - return { - variant: "viewer", - open: self.derive("mine", [store.open], ([all]) => openFor(all, principal)), - answer: ({ id, value }, ctx) => store.answer({ id, value, by: principal }, ctx), - }; -}); -``` - -The pending map with its resolvers is in-process state with promises, so it is a native of the store (opened and -closed with it). Its `close` rejects every waiter; askers retry with their key against the new store after a reload. - -:::review | Review: questions.md predates the v2 rules -`apps/p2/docs/questions.md` writes the store as `self.resource(...)` with a `replace` hand-over, which the design -rejected ("resources replaced in place"), and opens Chord call scopes in Durable hooks with `host.root`, which Q6 -replaced with `chord(fn, context)` (chapter XIV). Design §20 says the Questions store "must live in an object (it -predates natives only in objects)". The sketch above is this guide's translation, with the derive per viewer; filtering -per viewer in the store, once per variant, may be cheaper for large audiences. The `ask` signature passes typed -objects as arguments, which is fine (data), but the `audience` readable form cannot cross a link: across links, -audience must be data (principals) or computed by the store. -::: - -## Choosing - -| Case | Use | -| --- | --- | -| a multi-step login in one owner | a flow: the owner's job keeps the resolver, state shows the steps | -| a tool needs the owner's approval | Questions, `first`, audience the owner | -| the agent asks everyone present | Questions, `all`, audience from presence | -| deploy needs 2 of 3 approvals | Questions, `quorum(2)` | -| a notice (auth URL, device code, progress) | not a question: the asking work's own progress state | diff --git a/packages/chord/docs/next/explainer/src/13-registries.md b/packages/chord/docs/next/explainer/src/13-registries.md deleted file mode 100644 index 22efc7019..000000000 --- a/packages/chord/docs/next/explainer/src/13-registries.md +++ /dev/null @@ -1,185 +0,0 @@ - -# Extension Points {#registries lede="The facet, registries and the HTTP component: how plugins add behaviour to each other without callbacks, and how the owner stays in charge."} - -The notes plugin wants a download route, `GET /s/:session/notes.txt`. A logs plugin wants to time every request. -Several plugins add model providers, tools for the agent, commands for the TUI. Someone must combine these -contributions, decide conflicts, and replace one plugin's part as a whole when it reloads. Between plugins exactly two -things pass: object types and **registries**. - -## The facet, completed - -The facet has appeared since chapter II as the place that serves types. It is also itself a **pure object** with the -host's lifetime: - -```ts -// notes/server.ts -export default defineFacet((host, self) => { - const prefs = self.state(NotesPrefs); // plugin-wide documents - self.contribute(Http, { routes: { "GET /s/:session/notes.txt": exportNotes } }); - host.serve(NoteCountStore, …, { lifetime: "host" }); - host.serve(NoteCounts, …); -}); -``` - -- It may hold documents, references, routines and contributions; **no natives** (those need an object with its own - lifetime). -- Its identity is the plugin's name, so its persisted documents survive restarts and reloads. -- Served objects go live after the facet and may read and write its documents (chapter V). -- `host.serve` and `self.contribute` are valid only while the setup runs. The facet never `self.get`s a type it - serves. - -## Registries - -A registry is an extension point owned by someone (a Chord component, the app, a plugin). Each contributor -contributes **at most one value** per registry; the table is an immutable, ordered list of `(contributor, value)`; -the registry's `compose` turns it into the value consumers use. - -```ts -// owned by the HTTP component -export const Http = defineRegistry("chord.http", { - version: 2, - accepts: { 1: (old: HttpContributionV1): HttpContribution => upgrade(old) }, - compose: (table, { report }) => buildRouter(table, report), // pure; always delivers a value -}); - -self.contribute(Http, { routes: { … }, wraps: [ … ] }); // a contributor -const router = host.registry(Http).value; // a consumer: live -``` - -**`compose` runs at runtime, whenever the table changes:** after a reload's commit, when a contributor goes live or -ends, when a contribution changes. It is not a staging check: contributions can be dynamic (known only once a native -is open), so no check before a reload could be complete. `compose` always delivers a value with its fallback built in -(e.g. the first by plugin name wins) and reports conflicts through `report`, which become owner notices and telemetry -events. Readers get a plain value and never handle conflicts. Only if `compose` throws is the registry in an error -state, and readers get an error when they use it. Work that started earlier keeps the value it read. - -**Rules:** - -- **Who contributes** (R21): only facets and `lifetime: "host"` objects; only their `self` has `contribute` (a type - error elsewhere). Every object may read registries. -- **A contribution is in the table exactly while its contributor is live:** a facet's while its plugin is loaded; an - object's from `live` until it ends, fails or restarts. -- **Order** is sorted by plugin name: deterministic on every host and across reloads, and deliberately meaningless. - Meaningful order comes from the owner's composition (HTTP phases, below). Helpers: `keyed(table, entriesOf)`, - `ordered(table, itemsOf, orderOf?)`. -- **One authoritative declaration** (Q8): only the defining plugin's facet declares the registry to the host - (`compose`, version, `accepts`). Other plugins' tokens only identify it by name and types. A reload of the defining - plugin replaces the declaration atomically and the table recomposes. If the defining plugin is not loaded, the - registry has no value (readers get an error like a failed reference) and contributions wait in the table. -- **Versions** (Q8): a contribution carries the registry version its plugin was built against; older ones are adapted - by the defining facet's pure `accepts`. One without a matching version or adapter is left out and reported naming - plugin, registry and versions; its plugin keeps working. Contributions are in-process values (often functions) and - are not schema-checked. -- **Use sites guard contributions.** A contribution is untrusted code (third-party plugins are a goal). Every use site - catches throws and rejections when calling an entry, reports naming the contributor, and continues with the others. -- **The owner decides** conflicts, order and selection. Registries add behaviour; they never replace an owner's checks. - -## Natives through registries - -Registries are the only way a native reaches another plugin. An entry lives exactly as long as its contributor; -consumers look up at each use (or follow the composed value) and keep nothing from an entry beyond a use. The models -plugin's host-lifetime object contributes its runtime; the harness looks it up at every request. After a models reload -the next request gets the new runtime, while a running stream finishes on the old one. - -A handler that stores `const r = entry()` in a module variable and uses it for days has a bug: after the contributor -restarts it holds a closed native. - -## Contributions that change - -Some contributions are known only at runtime: MCP tools appear when a server connects. A contribution is either fixed -for its contributor's life (dynamic parts behind handles read at use) or a **readable**, which Chord subscribes to; -on every change Chord replaces that entry and runs `compose` again. For in-process values, `self.writable(initial)` is -a host-local readable of any value (no JSON, no wire, compared by identity) with `set`: - -```ts -const ext = self.writable(mcpExtension([])); // a Durable Extension, no tools yet -self.contribute(HarnessExtensions, ext); -self.routines.on(mcp.tools, (tools) => ext.set(mcpExtension(tools))); -``` - -There is no remove call. An entry leaves when its contributor ends, or after a reload whose new code no longer -contributes. A live contributor with nothing to offer sets its readable to the registry's empty value (an extension -without tools, empty routes); the table keeps exactly one entry per live contributor. - -## Optional dependencies - -Registries also replace optional references (chapter VI). Radius forwards connections to ports the ports plugin -manages, but ports is optional. So radius owns `radius.forwards` and ports contributes its forwards; without ports the -table is empty and radius works. - -## The HTTP component - -HTTP is an optional Chord component (`@earendil-works/chord/next/http`) built on a registry. A route gets a plain -request summary (method, path params, query, principal, provenance such as local or relayed, selected headers, parsed -and capped JSON bodies, uploads in chunks) and a `ctx` whose `principal` is the request's, so `ctx.get(Notes, …)` gives -exactly the view that person may see. The route returns a description; the component does the HTTP work once (ranges -with 206 and 416, HEAD, download headers with a safe filename, `nosniff`, CSP and its nonce, caching, streaming, abort -on disconnect): - -```ts -type HttpResponse = - | { kind: "json"; value: JsonValue } | { kind: "page"; title: string; body: string } - | { kind: "bytes"; name: string; mime: string; size: number; read(offset: number, length: number): Promise } - | { kind: "redirect"; to: string } | { kind: "error"; status: number }; - -// notes/server.ts -self.contribute(Http, { - routes: { - "GET /s/:session/notes.txt": async (req, ctx) => { - const notes = await ctx.get(Notes, { session: req.params.session }); // the requester's view - const text = new TextEncoder().encode(render(notes.notes.value)); - return { kind: "bytes", name: "notes.txt", mime: "text/plain", size: text.byteLength, - read: async (offset, length) => text.subarray(offset, offset + length) }; - }, - }, -}); -``` - -Routes compose with `keyed`: two contributors with one pattern is a reported conflict, and the first by plugin name -wins. **Wraps** are middleware in phases the owner defines (`observe`, `headers`, `transform`, outermost first), -within a phase in contributor order. No numeric priorities: plugins have no single registration order, so Fastify's -lifecycle phases and encapsulation are the model, not Express. - -```ts -// logs/server.ts -self.contribute(Http, { - wraps: [{ - name: "access-log", phase: "observe", scope: "all", // default scope "own": only own routes - wrap: async (req, next, ctx) => { - const t = Date.now(); - const res = await next(req); - log(req, res, Date.now() - t); - return res; - }, - }], -}); -``` - -`route` narrows a wrap to one route; a `route` that does not exist is a reported conflict. Wraps may add a security -boundary on top of an owner's checks, never instead of them. A request uses the router as it was when it started. -Request resolvers (who sent a request: cookie, loopback, relay) are hooks over the summary. - -## Channels and big bytes - -**Channels** are WebSocket byte channels (`send`, `onData`, `close`) for terminals, separate from object links and -with their own acknowledged windows, so a large state message never holds up keystrokes. - -**Big transfers are chunked by the owner,** not by Chord: bytes are a schema type (`bytes()`), carried natively -(never base64), and a 50 MB upload is a protocol of plain calls: - -```ts -beginUpload({ name, size }, ctx) → { upload } -append({ upload, offset, bytes }, ctx) // about 1 MiB per chunk; the awaited call is the backpressure -finishUpload({ upload }, ctx) → { id } -read({ id, offset, length }, ctx) → bytes -``` - -After a link drop the `append` in flight fails like any call; the writer asks the owner for the stored offset and -resumes. Unfinished uploads expire through the owner's routine. Chord has no stream type: it would need its own rules -for drops, cancellation, relays and restarts, which this offset protocol already survives. - -:::note | Built so far -Registries are layer L9. The HTTP component, channels and the chunked-transfer helper are deferred until after the -kernel and a first real proof (the harness, one tool with an approval pause, a code mode reload); until then the -playground serves its pages with plain app code. Their design stays the target. -::: diff --git a/packages/chord/docs/next/explainer/src/14-agent.md b/packages/chord/docs/next/explainer/src/14-agent.md deleted file mode 100644 index 40b69b555..000000000 --- a/packages/chord/docs/next/explainer/src/14-agent.md +++ /dev/null @@ -1,192 +0,0 @@ - -# The Agent {#agent lede="pi-durable inside Chord: the harness as a core object, models, environments and tools through registries, and how a tool reaches Chord objects and people."} - -Each Studio worker runs a pi-durable harness for its session. The harness must run for the worker's whole life, -whether anyone watches or not. Its models come from the models plugin, its execution environments from the envs -plugin, its tools from every plugin, and all of those reload independently. This chapter assumes you know pi-durable -and shows only how it plugs into Chord. - -## Shared foundations - -Durable and Chord v2 share modules: Durable imports Chord's `context`, `delta`, `json`, `output` and `sqlite`, and v2 -imports the same, so `Context`, `Draft` and `JsonValue` are one type each. A v2 readable is today's -`replicatedState` (`chord/src/state.ts`), so Durable's `viewState()` and document watches can be exposed as Chord state -members without conversion. Only plain data crosses between Durable and Chord objects. - -## The harness is a core object - -```ts -// studio/src/core/harness.ts (base code) -import { Harness as DurableHarness } from "@earendil-works/pi-durable"; - -export const ChordHostKey = createContextKey>("studio.chord-host"); -export const HarnessModels = defineRegistry("studio.harness-models", { version: 1, compose: firstModels }); -export const HarnessEnvironments = defineRegistry("studio.harness-envs", { version: 1, compose: firstEnv }); -export const HarnessExtensions = defineRegistry("studio.harness-extensions", { - version: 1, - compose: (table, { report }) => extensionSnapshot(table, report), // one Durable Extension per plugin -}); -const HarnessRuntime = defineRegistry<() => DurableHarness, () => DurableHarness>("studio.harness-runtime", { - version: 1, compose: onlyEntry, // core-private: one contributor -}); - -host.serve(Harness, (_input, self) => { - const info = self.get(SessionInfo, {}); - const harness = self.open(async ({ signal }) => { - const { id, dataDir } = info.init.value; - const base = withContextValue(ChordHostKey, host, withAbortSignal(signal, BACKGROUND_CONTEXT)); - return DurableHarness.open(await openNodeSqliteStorage(join(dataDir, id, "durable.sqlite")), { - models: () => host.registry(HarnessModels).value, // read at each use - env: (target) => host.registry(HarnessEnvironments).value(target), - settings: () => currentSettings(), - registry: durableRegistry(host.registry(HarnessExtensions)), // a RegistryReader over the composed value - }, base); - }, (h) => h.close(BACKGROUND_CONTEXT)); - self.contribute(HarnessRuntime, () => harness.value); // for core's conversation objects - return { … }; -}, { lifetime: "host" }); -``` - -- **Lifetime `"host"`:** it runs for the worker's whole life, also while nobody holds it, because Durable tasks - continue. -- **Base code:** it lives in the core facet, so a plugin reload never restarts it; only a base-code change (a host - restart) does. -- **Registries, looked up at each use:** Durable's `models` and `settings` accept functions (pi `7ec1f648e`), so each - generation gets the current value. After a models reload the next request uses the new runtime; a running stream - finishes on the old one. -- **Durable's storage is its own.** Chord documents never commit together with Durable transcripts. - -The design writes the call as `Durable.open(storage, { … })`; pi-durable's actual entry point is -`Harness.open(storage, options, context)`, used here as `DurableHarness` to keep it apart from the core object -`Harness`. - -## Start order - -At host start Chord loads every facet and waits until every facet and every `lifetime: "host"` object **other than the -harness** is live or failed: the **start barrier**. Then the registries hold the contributions of every plugin that -loaded, and the harness opens and resumes scheduling (R17). An object that must contribute at start, like the model -runtime, is therefore `lifetime: "host"`. Nothing waits for specific contributions; what is missing shows where it is -used: - -| Missing | What happens | -| --- | --- | -| a tool | its call gets an error result the model sees | -| a model | the generation reports "model unavailable" and waits, not fails, until a model appears | -| a task definition | the task stays blocked (`missing_task`) until the plugin is installed or the task is aborted | -| a plugin that failed | reported with its error; a facet in backoff contributes later and the registries recompose | - -The same holds when a session's database is copied to an instance with other plugins: missing tools, sections and -hooks are skipped, documents of missing plugins stay untouched, stored selections are never rewritten. The UI must -show blocked tasks with their reason and offer to abort them; a turn waiting on one hangs until then. - -:::review | Review: the harness exception to the barrier (C17, R17) -R17 makes the harness start after the barrier, a special case in the barrier's rule. Review round 3 proposed cut C17: -no harness-specific exception; the harness is an ordinary object the app's start code holds after `load` resolves. -Not decided. `chord-next-walkthrough.md` concept 28 still says the harness's activation *is* the start barrier, which -R17 replaced. -::: - -## Tools that call Chord - -Durable tools, hooks and sections receive Durable's `ToolExecutionApi` (or `HookApi`) and a `Context`, never a Chord -`ctx` (Q6). Core put the host into the harness's base context under `ChordHostKey`, and every phase context inherits -it. A core helper opens a Chord call scope from any Durable context: - -```ts -// studio/src/core/chord.ts -export function chord(context: Context, fn: (ctx: Ctx) => Promise): Promise { - const host = context.value(ChordHostKey); - if (!host) throw new Error("not inside the Studio harness"); - return host.root(fn, { signal: context.abortSignal, telemetry: context.telemetry }); -} -``` - -The scope runs as the host principal; attribution is data; references taken with `ctx.get` are released when `fn` -settles. There is no wrapping of tools and no app-specific `defineTool`. The notes plugin gives the agent a tool: - -```ts -// notes/worker.ts -const addNote = defineTool({ - name: "notes_add", // tools are named _ - description: "Add a note to this session's notes panel", - parameters: Type.Object({ text: Type.String({ maxLength: 2000 }) }), - execute: (args, api, context) => chord(async (ctx) => { - const info = await ctx.get(SessionInfo, {}); - const store = await ctx.get(NoteStore, { session: info.init.value.id }); - const id = await store.add({ text: args.text, by: `agent:${api.conversationId}` }, ctx); - return { output: [{ type: "text", text: `added note ${id}` }] }; - }, context), -}); - -export default defineFacet((host, self) => { - self.contribute(HarnessExtensions, defineExtension({ name: "@studio/notes", tools: [addNote] })); - … -}); -``` - -A reload of the notes plugin replaces its extension in the table; Durable pins a registry snapshot per task phase, so -a tool call already running finishes on the old code and the next phase uses the new one. - -## Tools that need a person - -Clients are not addressable, so a tool cannot call the user's tab. A tool that needs the user goes through an owner's -state: the Questions service (chapter XII). The model-facing `ask_user` tool and an approval hook both call -`Questions.ask`; tabs render the open questions and answer them. - -```ts -hook(ToolTask, { - beforeTool: async (call, api, context) => { - if (!needsApproval(call)) return undefined; - const decided = await api.memo("approval", context); // a rerun does not ask again - if (decided !== undefined) return decided ? undefined : { block: "Denied" }; - const allowed = await chord(async (ctx) => { - const q = await ctx.get(Questions, { scope: { session } }); - for (;;) { - try { - const r = await q.ask( - { kind: "confirm", message: `Run ${call.name}?`, allow: "Allow", deny: "Deny" }, - { audience: approvers, policy: { mode: "first" }, key: api.taskId }, - ctx, - ); - return r.answer; - } catch (error) { - if (ctx.signal.aborted) throw error; // a link blip is not a denial: retry - await backoff(ctx); - } - } - }, context); - await api.memo("approval", allowed, context); - return allowed ? undefined : { block: "Denied by the user" }; - }, -}); -``` - -Durable runs `beforeTool` in the tool task's call phase; after the execute checkpoint a crash retries execution -without hooks, so the stored decision covers that retry. The approval must cover the final arguments: put the -approval hook after hooks that rewrite arguments. - -## Conversations for the UI - -Chat panels need a conversation's live view, submit, steer, stop. The design leaves p3's conversation API to the p3 -design; one construction that follows the rules: a core `ConversationStore { conversation }` (local) opens the Durable -watch as its own native, getting the Durable harness from the core-private `HarnessRuntime` registry contribution (a -native may reach another object only through a registry entry), and exposes a bounded window as a derive. Per-principal -`Conversation { session, conversation }` views in front of it check who may send (chapter XIX). - -## Code mode - -Code mode is a plugin. It owns its `pi-codemode` sandbox as a native; its tool entry looks up the **current** sandbox -when `execute()` starts, so a call pinned by a phase before a reload (waiting in an approval hook, say) starts on the -new sandbox. On `reload` and `stop` the old sandbox is sealed and drained in `close`: running scripts finish or hit -their deadline, for at most 30 s (Q7). After a worker stop Durable replays as it does today. - -## What lives where - -| Data | Where | -| --- | --- | -| transcript-bound: entries, documents that fork and rewind with a conversation | Durable | -| app state: session list, notes, drafts, grants, settings | Chord documents | -| data core enforces on conversations (read-only one-shots, cwd followers) | a core per-conversation policy document, written when the conversation is created (`createConversation({ init })`) | - -**Telemetry:** pass the host's adapter as `HarnessOptions.telemetry`; Durable stores trace carriers on submissions and -tasks (chapter XVII). diff --git a/packages/chord/docs/next/explainer/src/15-reload.md b/packages/chord/docs/next/explainer/src/15-reload.md deleted file mode 100644 index 09ab349de..000000000 --- a/packages/chord/docs/next/explainer/src/15-reload.md +++ /dev/null @@ -1,131 +0,0 @@ - -# Reload and Rollout {#reload lede="What happens when you press Reload: per host, per object, and across the server, the workers and every tab."} - -Back to the preface. Mario changed the notes plugin's sort order and pressed Reload. Everything the earlier chapters -asked of plugin authors (synchronous factories, natives only in objects, routines instead of timers, data in -documents) exists so that this moment is safe. This chapter shows what Chord does with it. - -## On one host - -```text -1. load new bundles; an import or syntax error rejects -2. staging facet setups and every live object's new factory run with a recording self - (declarations only, nothing activated); any throw rejects -3. commit the host switches to the new code atomically -4. after per object: pure objects swap, native objects restart -``` - -**Rollback is all-or-nothing per host.** Any load or staging error leaves the whole host on its previous code, so a -host is always completely at one revision and its status is one value. Runtime states (a registry conflict, a failed -open, a migration) are not reasons to roll back; after the commit, errors stay with one object. There is no factory -trial, no migrate dry run, and no consumer contract check in staging: an incompatible consumer fails at bind with -`contract_mismatch` and a notice naming plugin and type. - -:::steps | Reloading @studio/notes on worker S1, object by object -+++ Load and stage -The new bundles load. The facet setup and the factories of `Notes` (three views), `NoteStore { S1 }` and `NoteInbox` -run with a recording `self`: they declare, nothing activates. Nothing throws, so the host commits to the new code. -+++ NoteStore: pure, swapped in place -`NoteStore` has no `self.open`. Its new methods and derives take over; `NoteList`, the drafts, its references and the -running import job stay. Routines restart, serialized: the new `cleanup` waits for the old handler in flight. The gate -opens. -+++ Running calls finish on old code -An `add` that was running when the swap happened finishes on the old code with its old handles; its transaction still -commits. Only new calls get the new handles. Exception (R15): if the reload raised a document's version, old code that -touches the migrated document throws, the document stays intact, and that call fails once. -+++ The import job keeps its old run -The job's `run` keeps the code it started with. When it settles, the **new** `finish` records the outcome. -+++ The views: pure, swapped -The three `Notes` views swap in place. Owner, Ann and Bob keep their references; the reader derive is recomputed with -the new sort and the difference is sent. Nobody sees a reconnect. -+++ NoteInbox: native, restarts -`NoteInbox` has a watcher. Its running work ends `aborted`, the watcher closes with reason `reload`, the new factory -runs, the watcher reopens. Its documents would stay. Whether a file dropped during the restart is seen is the -watcher's business: Studio's `watchDir` scans the directory once when it opens. -::: - -**Why natives restart instead of being handed over:** a native has no line, no transactions and no schema. If old and -new code shared it, they would interleave on it unchecked. In practice restarts are cheap: the Durable harness is base -code, terminal PTYs live in a separate terminal host process, and model runtimes, sandboxes and a search index rebuild -in milliseconds. Only reloading the plugin that owns SSH connections (running remote commands die) or the relay socket -(relayed connections drop) hurts. - -**Removing a type:** its objects end (routines stop, natives close, references end with an error); in-memory documents -go with them; persisted ones stay until retired. **Consumers elsewhere** see a restarting object like any object that -briefly goes away: `loading`, parked calls, then `live`. - -**UI facets reload too.** A reload unmounts and remounts components. That is why "leaving" (abort a login, close an -overlay) is never derived from an unmount (chapter VIII). - -## Across hosts - -A **revision** is (server incarnation, sequence) and names the complete desired code set: every facet with its -digest, not a batch of changes. Every install reconciles everything missing, and a host reports a revision only when it -runs that complete set. A revision also carries the code-set digest (a hash over the facet bundles' content hashes), -so a server restart with unchanged code triggers no reinstall. - -The order is **server, workers, clients**. Each host reports `installed`, `installed_failed` (with the objects that -failed) or `rejected` (with the reason; it stays on its previous code). A server that rejects stops the rollout. A -host that starts or reconnects installs the current revision before anything is routed to it. Installs are serialized -per host and bounded (60 s). - -Calls keep flowing during a rollout. When a call reaches a provider on another revision, the mismatch response -carries the provider's revision and status: - -| Provider | The call | -| --- | --- | -| ahead, or behind and installing | `updating`: values readable, calls park (with the park deadline) | -| rejected | fails at once; retried on the next revision | -| same revision | `contract_mismatch`: a real mismatch, final | - -A host checks its own install status first: if its own install was rejected, a call fails at once with the reason -instead of waiting on a provider. Calls parked on a binding that is released end with an error and are not resent. -There are no cross-host transactions and no votes. - -:::sequence animate | A tab calls into a worker that is still installing. The call parks and runs once the worker reports the revision. -actor tab "tab" "revision 8" -actor server "server" "revision 8" accent -actor worker "worker S1" "installing 8" blue -server => worker: install revision 8 -tab -> server: call Notes.add (new contract) -server -> worker: relay -worker --> server: updating (rev 7, installing) -server --> tab: updating -tab -> tab: park (deadline 60 s) -worker --> server: installed 8 -server --> tab: provider at 8 -tab -> server: call Notes.add (resent from park) -server -> worker: relay -worker --> tab: result -::: - -## Start barrier - -At host start and first load there is one more rule (chapter XIV): load every facet (the core facet with its start -data among them), wait until every facet and `lifetime: "host"` object other than the harness is live or failed, then -resume dependent work. Plugin reloads need no barrier. - -## Loading and the dev loop - -- **Roots:** plugins come from built-in, global (`/plugins`), package (npm or git, installed with - `--ignore-scripts`) and project roots, found by directory listing; no source parsing. The server selects which load. - Dependencies between plugins resolve by contract name at runtime. Names are prefixed: objects `.`, - tools `_`. -- **Loader D:** one base bundle per surface (host packages and the app runtime) plus one script per facet, with host - packages passed in (no module map). Old generations are reclaimed by the garbage collector; a stable filename per - (facet, digest), and V8's compilation cache off on hosts that reload in place. The loader is an adapter: static - imports (tests, Durable Objects) work the same, with the app passing the plugin name to `load`. -- **Dev loop:** a source watcher marks plugins dirty and never applies anything. A reload control (the playground's web - shell, a CLI, an agent tool such as p2's `reload_plugins`) reloads the changed plugins, or restarts all hosts for - base code. Reloads are always explicit. - -:::review | Review: project plugins (#14, deferred) -Per-directory plugins with trust prompts and snapshots (`apps/p2/docs/plugin-loading.md`, still the reference for -roots, trust and scopes) need their own pass over this design. Until then, treat the project root as designed but not -settled. -::: - -:::note | Built so far -Reload and rollout are layer L10, the last before telemetry. The frozen kernel has a reload path with host values and -a development freeze, both removed from the design. -::: diff --git a/packages/chord/docs/next/explainer/src/16-contracts.md b/packages/chord/docs/next/explainer/src/16-contracts.md deleted file mode 100644 index 51e0126cb..000000000 --- a/packages/chord/docs/next/explainer/src/16-contracts.md +++ /dev/null @@ -1,310 +0,0 @@ - -# Changing Contracts {#contracts lede="How a contract gets a new version while old tabs, old workers and old plugins are still talking to it."} - -Chapter XV ended with a tab that calls into a worker on another revision. That case is short-lived: the rollout -finishes and both sides run the same code. Other cases last longer. A phone kept a tab open overnight with yesterday's -web bundle. A plugin from another author was built against `Notes` version 1 and is not updated yet. In both, a -consumer holds a contract that the provider no longer has. This chapter is about what the provider does then. It is -also the one part of Chord v2 that is built: everything in the sections on `evolve`, `checkAccepts` and fingerprints -runs today in `packages/chord/src/next/contracts/`. - -## What has a version - -Everything that is stored or crosses a boundary has a version. Each surface handles an old version its own way: - -| Surface | Version | When an old one shows up | Chapter | -| --- | --- | --- | --- | -| object type (contract) | `version` plus fingerprint | `evolve` and `checkAccepts` on the provider; otherwise `contract_mismatch` | this one | -| document | `version` plus schema fingerprint | pure `migrate` on access; a schema change without a bump fails the object `broken` | V | -| job | `version` | `accepts` on outcomes and `running`; `blocked` without an adapter | XI | -| registry | `version` | `accepts` adapts contributions; one without a match is left out and reported | XIII | -| native | none | the object restarts on reload | IX | -| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | -| link protocol | protocol version | refused; hosts update together, a web page reloads | XVII | -| UI components, Chord and app APIs | none yet | exposed as they are; an SDK will version them later | | - -The rule behind every row is the same as for documents: **every change bumps the version**, additive ones included. -Chord never guesses compatibility from the shape of two schemas. - -## A history, not a file of versions - -A contract history starts with a **base**: a full `remote()` or `local()` declaration with an explicit `version`. The -`Notes` contract from chapters II and III is a base at version 1. Studio now wants pinned notes: notes get a `pinned` -flag and editors get a `pin` method. Instead of editing the declaration, the plugin continues the history: - -```ts -// notes/contract.ts -import { bool, evolve, nul } from "@earendil-works/chord/next"; - -export const NoteV2 = obj({ id: str(), text: str({ max: 2000 }), by: str(), at: int(), pinned: bool() }); -export const PublicNoteV2 = obj({ id: str(), text: str(), at: int(), pinned: bool() }); - -export const NotesV2 = evolve(Notes, (e) => - e.state("editor", "notes", obj({ items: arr(NoteV2) })) - .state("reader", "notes", obj({ items: arr(PublicNoteV2) })) - .addMethod("editor", "pin", obj({ id: str(), pinned: bool() }), nul()), -); -``` - -`evolve(prev, (e) => …)` appends one version to `prev`'s history: `NotesV2.version` is 2, and `NotesV2` still knows -version 1 exactly as it was declared. Every version is **materialized and fingerprinted at declaration**. That is what -makes old code safe: an old plugin bundle that contains its own copy of `Notes` version 1 computes the same -fingerprint as the history inside `NotesV2`. There is no build step, no generated file and no fingerprint copied by -hand. - -The edits: - -| Edit | Changes | Kind | -| --- | --- | --- | -| `e.state(variant, member, schema)` | one state member of one variant | remote | -| `e.addState(variant, member, schema)` / `e.removeState(variant, member)` | adds or removes a state member | remote | -| `e.state(member, schema)`, `e.addState(member, schema)`, `e.removeState(member)` | the same, flat | local | -| `e.args(name, schema)` / `e.result(name, schema)` | one side of a method | both | -| `e.method(name, args, result)` | both sides of a method | both | -| `e.addMethod(variant, name, args, result)` | adds a method (or an existing one to another variant, same signature) | remote | -| `e.addMethod(name, args, result)` | adds a method | local | -| `e.removeMethod(name)` | removes a method from every variant | both | - -Edits chain. The `method(...)` wrapper appears only in base declarations; edits take the schemas directly, so a -method without a result is added with `nul()`. The rules of chapter III still hold across versions: a method has one -signature in every variant, and a member's kind never changes, not even after it was removed and added again. - -**What `evolve` cannot do:** change the input or the set of variants. Both change what an object *is* (its identity, -who gets which members), and no adapter can turn one into the other. That needs a new history base, below. - -## The provider says what it accepts - -`NotesV2` says what changed. It does not say how version 1 consumers are served; the **provider** decides that, with -mappings that live in its own code, never in the contract. The worker facet declares them with `checkAccepts`: - -```ts -// notes/worker.ts -import { checkAccepts } from "@earendil-works/chord/next"; - -const toV1 = ({ pinned, ...note }: Static) => note; -const toPublicV1 = ({ pinned, ...note }: Static) => note; - -const notesAccepts = checkAccepts(NotesV2, { - 1: { - state: { - editor: { notes: (current) => ({ items: current.items.map(toV1) }) }, - reader: { notes: (current) => ({ items: current.items.map(toPublicV1) }) }, - }, - methods: {}, // add and remove did not change; pin is new - }, -}); -// notesAccepts.versions: [{ version: 1, fingerprint: Notes.fingerprint }] - -host.serve(NotesV2, notesFactory, { accepts: notesAccepts }); // working name for the objects-layer option -``` - -`checkAccepts` is **pure**: it checks the mappings without calling them and returns only the version metadata of the -old versions it lists. A provider that lists nothing accepts only its current version. The objects layer later -compares a consumer's version and fingerprint against this list. - -The mappings point in the direction the data flows: - -| Member in the old version | Mapping | Direction | -| --- | --- | --- | -| state, changed | `(currentMember) => oldMember` | provider's current value to what the old consumer expects | -| state, removed | `(currentVariantState) => oldMember`, or `"refuse"` | receives the whole current state of that variant | -| method arguments, changed | `args: (old) => current` | old caller's arguments to current ones | -| method result, changed | `result: (current) => old` | current result back to the old shape | -| method, changed or removed | `{ call: async (oldArgs, ctx) => oldResult }`, or `"refuse"` | replaces the method for old callers | -| member added since | nothing | an old consumer never asks for it | - -A changed state member cannot be refused: the old consumer still has that member and must get a value. A removed one -can be computed from what the variant has now, or refused. `e.method(name, args, result)` replaces both sides and -therefore needs both mappings, even if one side stayed the same (then an identity mapping). - -**Required, at type level and again at runtime.** For every member that exists in the old version and changed or was -removed since, the mapping is required by the type of `checkAccepts`. Because plugins are bundled without type -checks, the same check runs at declaration: a missing mapping, a key for a member that needs none, an unknown version -or a wrong shape throws, naming the member's path (for example `accepts[1].state.reader.notes`). An absent `state` or -`methods` section counts as empty and may be left out only when nothing there needs a mapping. - -**Conservative with no-op edits.** An edit that sets a schema to what it already was, or changes a member and then -changes it back, still requires an identity mapping. TypeScript erases constraints such as `{ max: 2000 }` from the -static type, so two schemas that look the same to the compiler may differ; Chord does not try to tell. - -### Old versions map straight to the current one - -Studio later replaces `remove` with `archive`, which keeps the note in a trash list: - -```ts -export const NotesV3 = evolve(NotesV2, (e) => - e.removeMethod("remove").addMethod("editor", "archive", obj({ id: str() }), nul()), -); - -const notesAccepts = checkAccepts(NotesV3, { - 1: { - state: { editor: { notes: (c) => ({ items: c.items.map(toV1) }) }, reader: { notes: (c) => ({ items: c.items.map(toPublicV1) }) } }, - methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } }, - }, - 2: { methods: { remove: { call: async ({ id }, ctx) => { await archiveNote(id, ctx); return null; } } } }, -}); -``` - -Each entry maps **one old version directly to the current one**, not through the versions in between. Version 1 -needs the state mappings (its notes have no `pinned`) and the `remove` replacement; version 2 needs only the -`remove` replacement, and leaves `state` out because its state is unchanged. A member that an old version never had -needs nothing for that version, even if it was added, changed and removed later. Dropping support for version 1 is -deleting its entry: version 1 consumers then get `contract_mismatch`. - -## Binding an old consumer - -A hold carries only the consumer's contract **version and fingerprint**, never a description of the contract (R6). -The provider decides with its own schemas: - -:::flow wide animate | What a provider does with a hold. The decision uses only the provider's own history; a peer's schema is never an input. -row "The hold arrives" -box hold "hold" "version 1, fingerprint F1" blue -box cur "equal to current?" "version and fingerprint" -box acc "accepted old version?" "F1 from the provider's own history" -row "Outcome" accent -box live "bind" "current members" -box adapted "bind adapted" "through the mappings" accent -box mismatch "`contract_mismatch`" "or updating, during a rollout" muted -arrow hold -> cur -arrow cur -> live "yes" -arrow cur -> acc "no" -arrow acc -> adapted "yes" -arrow acc -> mismatch "no" -note "A mismatch fails only that consumer's binding. The provider object and its other bindings are not affected." -::: - -Once a version 1 tab is bound, it is served as if version 1 were current: - -:::steps | A version 1 tab on a version 3 worker -+++ Bind -The tab holds `Notes { S1 }` with version 1 and fingerprint F1. The worker's history computes F1 for version 1, and -`notesAccepts` lists version 1, so the hold binds adapted. The tab gets the `editor` variant as version 1 defined it: -`notes`, `add`, `remove`. It never sees `pin` or `archive`. -+++ State through the mapping -Chord serves the adapted state like a derive of the provider's state: it applies the mapping to the current value -and keeps the result per binding. The snapshot the tab receives has no `pinned` fields. -+++ A commit -The owner pins a note from a current tab. The worker sends the current tab the draft's ops. For the version 1 tab, -Chord applies the mapping to the new value and sends the difference to its last adapted value (`diffRevisions`). The -mapping produced nothing new for a pin, so the version 1 tab receives no change at all. A large list is never resent -whole on every commit. -+++ A call -The tab calls `remove({ id })`. Its arguments are checked against the provider's own version 1 schema, then the -`call` replacement runs `archiveNote` on version 3 code. Its `null` result is what version 1 expects. Had the method -only changed, `args` would have mapped the arguments forward and `result` the result back. -+++ The worker moves on -A later revision drops the version 1 entry. When the tab re-holds (or the worker restarts and the tab binds again), -the hold gets `contract_mismatch`, and Studio's panel shows "reload to update" instead of failing silently. -::: - -## A new history base - -Some changes cannot be adapted. Studio decides that notes belong to a project, not a session: the input changes from -`{ session }` to `{ project }`. That is a new base, written in full, with an explicit version that continues the -published numbering: - -```ts -export const NotesV4 = remote("notes", { - version: 4, // explicit: the published history used 1 to 3 - input: obj({ project: str() }), - placement: () => Server, - variants: { - editor: { state: { notes: obj({ items: arr(NoteV2) }) }, methods: { add: method(…), pin: method(…), archive: method(…) } }, - reader: { state: { notes: obj({ items: arr(PublicNoteV2) }) } }, - }, -}); -// checkAccepts(NotesV4).versions is empty -``` - -A new base has no history before its version and **cannot accept older versions**, not even for an additive change. -Every consumer built against versions 1 to 3 gets `contract_mismatch` and must update. Later `evolve` calls continue -from version 4. The same applies to a change of the variant set. Do not spread an older declaration into a new one or -copy its fingerprint; the contract objects are branded and `evolve` and `checkAccepts` reject copies. - -## Local contracts - -`local` contracts (chapter IV) evolve the same way, with a flat members map and no variants: - -```ts -export const SessionInfoV2 = evolve(SessionInfo, (e) => e.state("init", InitV2)); -checkAccepts(SessionInfoV2, { 1: { state: { init: (current) => toInitV1(current) }, methods: {} } }); -``` - -Same-host consumers of a `local` type are usually reloaded together with their provider, but not always: another -plugin on the same host may be built against an older version. The rules are identical; only the variant argument is -missing. - -## Implementing a contract - -`Implementation` is the type of everything a provider must supply for the current version: one state -getter per member and variant, and one asynchronous method per method, each receiving `(args, ctx)`: - -```ts -const impl: Implementation = { - state: { editor: { notes: () => list.value }, reader: { notes: () => publicList.value } }, - methods: { add: async ({ text }, ctx) => addNote(text, ctx), remove: async ({ id }) => removeNote(id), pin: async (a) => pin(a) }, -}; -``` - -Keep declaration types **inferred** (no annotation on `NotesV2`): the inferred type carries the complete history, and -`checkAccepts` needs it to know what changed. - -:::review | Review: L1's implementation shape and the factory's return value -`Implementation` in `src/next/contracts/types.ts` is `{ state: { variant: { member: getter } }, methods }`. The -design's factories (`design.md` §6, used throughout this guide) return `{ variant, member: value, method: fn }`. -The objects layer (L4) has to connect the two, and also define how `serve` receives the checked accepts (this guide -uses the working name `{ accepts }` as a third argument) and what `"refuse"` does at runtime. The natural reading, -used in the stepper above, is that a refused method call and a hold that needs a refused state member end with -`contract_mismatch`; no source says so. -::: - -## The fingerprint - -The fingerprint is a 64-bit hash (`hash64`, portable across runtimes) of a **normal form** of the contract's -schemas. The normal form decides what counts as a change: - -| Ignored | Significant | -| --- | --- | -| order of object keys, `required` and `enum` (sorted) | order of union and intersection branches (`anyOf`, `oneOf`, `allOf`) and of tuples | -| annotations on schema nodes: `title`, `description`, `examples`, `$comment`, `deprecated` | defaults, formats, patterns, bounds, `$ref` names | -| placement | input, variants, member kinds and every member schema | - -Union order is significant because it is meaning: TypeBox picks defaults and decodes by the first matching branch. -Annotations are dropped only on schema nodes; an object property that happens to be named `title` is data and counts. - -Only data schemas are allowed. One walker copies, validates and normalizes each schema against a **whitelist of -TypeBox keywords**: primitives, arrays and tuples, objects and records, unions and intersections, `$defs` and `$ref` -(use `$ref` for recursive types), constraints and annotations. Conditional and dependent schemas, `not`, -`definitions`, `contentSchema`, `unevaluated*`, `contains`, `propertyNames`, unknown keywords, executable or custom -TypeBox features, arbitrary `Unsafe` and refinements, accessors, symbols and cyclic objects are rejected at -declaration. Bytes are the one special leaf: `bytes({ min, max })` checks a `Uint8Array` and has the data marker -`{ type: "bytes", minLength?, maxLength? }`, so it hashes like any schema; TypeBox's checking still works on it. - -Schemas are deep-copied into a private, frozen history; your own schema objects are copied, not frozen. -`NORMAL_FORM_VERSION` is part of every digest, so a change of the normal form is a link protocol change. - -**Not a security boundary.** A fingerprint says "we were compiled against the same contract"; it proves nothing. A -hostile client can send any fingerprint it likes. Providers therefore always validate arguments with their own -schemas (chapter IV), whatever the fingerprint says. - -## Rules of thumb - -- Treat a published contract like a published API: every change is `evolve`, every `evolve` needs a decision about - old consumers. -- Keep mappings small and pure. A state mapping runs for every commit and every adapted binding; make it a map over - the value, not a query. -- Accept the versions your users still run (one or two back during a normal rollout), and delete entries when the - metrics show no old bindings. -- Change the input or the variants only with a new base, and only when updating every consumer is acceptable. -- Contract, document and job versions are separate and each is decided on its own. Pinned notes needed `NotesV2` - for the wire and `NoteList` version 2 with a `migrate` that adds `pinned: false` (chapter V), but no new - `ImportJob` version, because its outcome did not change. - -:::note | Built so far -This chapter's contract layer is built: `packages/chord/src/next/contracts/` (commit `7d2bb8ea8`, about 1,300 lines -with `remote`, `local`, `method`, `evolve`, `checkAccepts`, `Implementation`, the schema helpers, `fingerprintOf`, -`schemaNormalForm` and `NORMAL_FORM_VERSION`), with nine test files covering evolution, fingerprints, the hash, -local contracts, long chains, schemas, types and review findings. It stands alone: it hosts nothing, calls no -mapping and does not import the frozen runtime. Binding by version and fingerprint, adapted state and calls belong to -the objects and protocol layers, not built yet. -::: diff --git a/packages/chord/docs/next/explainer/src/17-wire.md b/packages/chord/docs/next/explainer/src/17-wire.md deleted file mode 100644 index 67bd21118..000000000 --- a/packages/chord/docs/next/explainer/src/17-wire.md +++ /dev/null @@ -1,251 +0,0 @@ - -# On the Wire {#wire lede="What travels over a link, what a relay does with it, how big things are kept in check, what each hop costs, and how to see it in traces."} - -Most plugin code never thinks about the wire: it declares state, commits transactions and calls methods, and Chord -moves the bytes. You need this chapter when something is slow, when a value is large, when a link misbehaves, or when -you want to read a trace. It also explains why the earlier chapters insisted on small operations, shared structure and -no callbacks: each of those rules removes a cost on some hop. - -## What a state member sends - -Every exposed document, derive or readable is a **state member**. Over a hold it produces one of these: - -| Update | When | -| --- | --- | -| `snap` | on hold and on re-hold: the whole value, once | -| `ops` | per commit: the draft's recorded ops, no diff | -| `ops` from a diff | per change of a derive (and of an adapted binding, chapter XVI) | -| throttled `ops` | job progress | -| `reset` | a migration commit replaced the value (adapted for older consumers) | -| `error` | the member failed | - -**Commits are atomic for consumers.** On the same host, subscribers are notified once, after every document of the -commit is adopted. Remotely, one `state` message per hold carries a batch of member updates that is applied before -listeners run, and a commit that touches several holds on one link travels as **one** `state` message on that link. -A consumer never sees half a commit. - -## Messages - -A message is a **header plus opaque parts** (R1). There is one part per hold the message concerns; the header says -which hold each part belongs to and how large it is. - -| Direction | Kind | Parts | -| --- | --- | --- | -| consumer to provider | `hold`, `release`, `call`, `cancel` | `call` has one (the arguments) | -| provider to consumer | `bound` | one: the snapshot | -| | `state` | one per hold the commit touched on this link | -| | `result` | one | -| | `gone` | none | - -There are no request ids in application code. Hold ids and call ids exist only inside the protocol, per link. - -**One binary format on every link (Q4):** `u32 header length | header JSON | part bytes`, as one `Uint8Array`. A -WebSocket sends it as a binary frame, IPC passes it with `serialization: "advanced"`, a memory link delivers it -asynchronously. Encoding, framing, header checks, id resolution, flow control and size limits are one shared protocol -layer; link adapters only move bytes. Tests on memory links therefore run the production path. - -A part is encoded **once, at its origin**, with that hold's own Delta encoder (paths are interned per hold, end to -end between provider and consumer). A JSON part above a threshold (about 16 KiB) is compressed once at its origin, -whatever the link; byte segments stay raw; relays forward compressed parts as they are; the header names the codec. - -**The header is a TypeBox schema (R4)**, a union tagged by `kind`; the protocol types derive from it. Headers from -untrusted links (client WebSockets) are checked with the schema compiled once; trusted links (IPC between server and -workers) are not checked. **Unknown ids are ignored on both sides (Q1).** They are normal races: a `result` crossing a -`cancel`, a `call` crossing a `gone`, a state part crossing a `release`. The side that ended a hold or call settles -its own pending work: a consumer that receives `gone(h)` rejects its pending calls on `h`, and the order per -direction guarantees nothing else for `h` follows. Only a header that fails the schema is a protocol error, and it -closes that link. - -The link protocol has a version. Hosts with different protocol versions refuse each other; a web page reloads to get -the server's protocol. - -## Through a relay - -A tab never talks to a worker directly (chapter I). The server **relays**, and a relay reads and rewrites only -headers: it maps hold and call ids between its two links, stamps the principal on `hold` and `call` (a client's -principal field is always overwritten; only a trusted parent link may carry one), and groups parts per destination -link, so each link gets only its own holds' parts. It never parses a payload. - -:::sequence animate | The owner adds a note while Ann watches. The commit leaves the worker as one `state` message with two parts; the server splits it by destination without decoding either part. -actor owner "owner's tab" "hold h1" blue -actor ann "Ann's tab" "hold h2" muted -actor server "server" "relay" accent -actor worker "worker S1" "NoteStore, Notes" -owner -> server: call c7 Notes.add (1 part) -server -> server: map ids, stamp owner -server => worker: call c3 (part forwarded as is) -worker -> worker: method runs, self.tx commits -worker => server: state: parts for h1', h2' -server -> server: group parts per link -server --> owner: state (h1 part) -server --> ann: state (h2 part) -worker --> server: result c3 -server --> owner: result c7 -::: - -The `state` message reaches the owner's tab before the `result`, so when `await notes.add(…)` resolves, the new note -is already in `notes.value`. Ann's view is a different variant (reader) and gets its own part from its own derive; the -relay neither knows nor cares. - -## Flow control - -Each link has **one byte window**, counted from each message's `byteLength` (parts are already encoded bytes; nothing -is measured or stringified to count it). The window covers every message, first snapshots included. - -- **Snapshots are always admitted (R19)**, even when larger than the window; they use it up. Whether a hold - **overflows** is decided on the changes queued after the snapshot. -- **Under backpressure** a commit's ops are collected per hold and encoded into one part when credits return. The - consumer still gets the commit atomically. -- **After an overflow** the hold is dropped and established again once the previous snapshot's debt has drained - (credits back at the full window, Q3), and not more often than a minimum interval. So there is at most one snapshot - in flight per hold: no re-hold storm, and no stall for a snapshot larger than the window. -- **Calls** are throttled by call credits the provider grants (a generous window). Beyond them calls wait locally. - Nothing is refused for being "too many", and holds are not limited by count; authorization and input validation - protect them. -- **Byte-heavy traffic** such as terminals uses separate channels (chapter XIII), so a large state message never - holds up keystrokes. - -**Recovery:** after a link drops, the consumer re-holds every hold over the new link and receives snapshots; calls in -flight fail as on any disconnect, calls made meanwhile park. A relay whose window overflows releases the hold and -re-holds; the provider sends a fresh snapshot. After a routed link drops, Chord re-resolves the address (with the -per-address backoff) while holds on it exist, so a passive viewer recovers without anyone pressing anything. - -## Size limits - -Limits apply **per part** (R2, Q2), both to the compressed and the decoded size, and are checked from the sizes in the -header before anything is decoded; zlib's `maxOutputLength` enforces the decoded limit in the one decode function. - -| Situation | What fails | -| --- | --- | -| a part over the limit | only its own hold or call, with `too_large`; the other parts of the message apply | -| a snapshot over the limit | its hold ends `failed`, with no automatic re-hold | -| a WebSocket frame over `maxPayload` (e.g. 64 MiB) | the connection closes (code 1009); this is the transport's hard cap | - -Defaults for client WebSockets are 16 MiB compressed and 64 MiB decoded per part, configurable per link kind. The -decoded limit travels with relayed work: the server stamps the admitting link's limit on the header at admission -(never taken from the client), and the host that decodes enforces it. IPC therefore needs no limit of its own. - -Large data is never a large state member or a large argument. Files go through chunked transfers or the HTTP -component (chapter XIII); long lists are keyed documents or bounded windows (chapter VIII). - -## What each hop costs - -Chord v2's goal is as little compute and memory as possible while staying correct: **O(size of the change)** per -commit and per viewer, independent of the size of the value. A 500 KB value is touched as a whole only when a new -viewer takes its first snapshot. Delta, the value library under documents, was built for this, and its own primitives -are almost free: `diffRevisions` on shared subtrees takes 0.018 ms, `applyImmutable` of a small change 0.004 ms. - -| Hop | Work | -| --- | --- | -| commit | Delta's `prepare` (copies and checks only the values placed in this commit), adopt by pointer swap; storage writes the delta | -| reads in a transaction | the draft overlay, no copy; an unchanged draft writes nothing | -| subscribers on the same host | the same value and ops, by reference | -| derives | compute plus `diffRevisions`; unchanged parts stay the same objects (`mapShared`, chapter VIII) | -| send | one encoding per hold part at its origin, size taken from the encoded bytes | -| relay | header only: ids mapped, principal stamped, parts grouped; parts never parsed | -| receive | compiled header schema on untrusted links; arguments stripped and validated once at the provider (compiled schema); inputs normalized | -| apply | `applyImmutable`, structure shared | -| new hold | the value encoded once (unavoidable) | -| memory | one authoritative value per document per host; replicas and derived values share structure; one cached value per adapted binding | - -These are targets. The frozen kernel measured far worse, which is why the design is so strict here: - -:::stats -- 35 ms | one small change to a 500 KB value in the frozen kernel, no viewer -- +13 ms | per viewer on top (10 viewers: 130 ms) -- 17 / 33 / 101 ms | a tiny SQLite commit after 0 / 10 / 50 earlier deltas -::: - -Nearly all of that came from layers around Delta: whole-value validation on every access, whole-value freezing, -JSON-ness walks on every received frame, double stringification to measure a frame, snapshots cloned per viewer, and -a SQLite storage that replayed history, and schemas rebuilt for every check (a compiled TypeBox check takes -0.012 ms, an interpreted one 5.6 ms). The redesign removes each of them (chapters V, VIII and this one); the -before/after per hop is in `apps/p2/docs/p3-reviews/v2-cost-model.md`. - -:::review | Review: v2-cost-model.md is partly outdated -The measurements and the per-hop "after" targets in `apps/p2/docs/p3-reviews/v2-cost-model.md` are current. Its -closing section "Decisions this needs" is not: items 6 and 7 (validation by the caller's trust; contract descriptions -once per link and fingerprint) were replaced by R7 (validate for every caller) and R6 (no descriptions on the wire), -and §10 still describes the description fetch. -::: - -### Cost guards - -Chord reports **full-value walks, deep copies, encodes, schema validations and materializations** as ordinary -telemetry metrics, always on; with the NOOP adapter they cost an empty call where such work happens. Tests use the -in-memory adapter and assert them per hop. The reference case: one appended note to a 500 KB document with 10 viewers -does no walks, no copies, one encode per hold part at its origin and none at relays, and no validations. A -`vitest bench` suite measures the hops. - -### Ways to make it slow anyway - -| Mistake | Why it costs | Instead | -| --- | --- | --- | -| a derive that builds new objects for unchanged data | every commit diffs and sends O(value) | keep unchanged items identical: `mapShared`, `previous` (chapter VIII) | -| one long array that changes often (R24) | sharing is per path; each change copies one slot per element and visits each | keyed documents or a bounded window | -| bytes as base64 strings | inflated, parsed, compressed as text | `bytes()` leaves; chunked transfers for files | -| awaiting I/O inside `self.tx` | the object's line is blocked for every caller | do the I/O first, then a short transaction | -| storing registry entries or references in documents | stale after a reload, not data | look up the current value at use | -| opening natives at module level or in a facet | no owner: never closed, never restarted | `self.open` in an object (chapter IX) | -| callbacks as arguments | cannot cross a link, keep old code alive | registries, jobs, flows | -| one big object that everybody views | one large snapshot per viewer and per reconnect | split by viewer need; views of the same variant share one value | - -## Telemetry - -Chord reports traces, events and metrics through `@earendil-works/pi-telemetry`. Each host gets **one adapter**, -`createHost({ telemetry })`, NOOP by default; the adapter decides sampling, buffering, export and filtering. Only hosts -hold the adapter, to start root spans. Code sees spans: - -- `ctx.telemetry` is the current span (`Ctx` extends `Context`, so it is the same field Durable and pi-ai read). Every - span carries its adapter's metrics and propagation, so code with a `ctx` can record a metric or inject a carrier. -- `ctx.trace(name, attributes, fn)` runs `fn` in a child span with a child `ctx`. -- App values such as the principal stay out of Durable's context. - -**Across links**, `call` and `hold` carry the caller's trace carrier as a bounded header field (R5: limited key count -and string lengths); the provider runs the method under it, and each relay adds a `chord.relay` span. The call from the -sequence above, as a trace: - -```text -chord.call Notes.add [owner's tab] -└─ chord.relay [server] - └─ chord.method Notes.add [session:S1] principal=owner - · event chord.commit docs=2 ops=5 -``` - -Adapters trust typed input: no validation of malformed values and no try/catch around recording. Only carriers from -another process are checked (keys present, strings), and a bad one falls back to a new root. - -**Shared work** (one activation serving several `get`s, a worker start) is its own root span, linked to the trigger -that started it; each waiting span links to it. Routine runs, job runs, eviction, backoff retries and background -closes are root spans too, linked to their trigger where there is one. - -| Tier | What | -| --- | --- | -| spans | calls, methods, relays, holds, activations, opens, closes, loads, revocations, resolves, routine runs, job runs | -| events | commits inside methods, retries, parking | -| metrics | state messages, WebSocket bytes, commit durations, in-flight calls, active holds, open links, park durations, backpressure waits, the cost guards | - -Ids are attributes of spans, never of metrics. **Nothing is measured or walked for telemetry**; only sizes the code -already has are reported. Errors carry a fixed category and a message cut to 500 characters; cancelling is not an -error. Chord ships a typed telemetry schema, as pi-ai (`AI_TELEMETRY_SCHEMA`) and Durable -(`DURABLE_TELEMETRY_SCHEMA`) do. - -**Storing and reading:** `@earendil-works/chord/telemetry-sqlite` over `chord/sqlite` (Node and Cloudflare) stores one -row per settled span (events and links inline) and metrics aggregated per series per 10 s, flushes in batches every -second, keeps 7 days, and drops (and counts) beyond a bounded queue. The read interface `TelemetryStore` is shared: -`findTraces` (by name, attributes, errors, time window, with limit and cursor) returns complete traces, `getTrace` -one, `findMetrics` metrics. One renderer, `pi-telemetry/render`, prints traces as above; it prints links and never -follows them. The playground's trace viewer reads the same store. - -**Durable** already emits submit and phase spans, stores carriers on submissions and tasks (`TaskOptions.trace`), -and pi-ai emits `pi.ai.request` and `pi.ai.poll`. With the harness given the host's adapter (chapter XIV), an agent -tool that calls `Notes.add` through `chord(fn, context)` shows up as one tree from the model request down to the -commit. - -:::note | Built so far -Nothing of the protocol is built in v2: messages, relays, flow control and limits are layers L6 and L7, telemetry in -Chord is L11. Built in pi: `pi-telemetry` (propagation, links, metrics, the store interface, the renderer), -`chord/context` with `Context.telemetry`, pi-ai and Durable telemetry, and `chord/telemetry-sqlite`. Delta is built; -`mapShared` and bytes as atomic leaves in `diffRevisions` are still to do. The frozen kernel sends descriptions on the -wire and closes a link on an unknown id; both are replaced. -::: diff --git a/packages/chord/docs/next/explainer/src/18-p2-core.md b/packages/chord/docs/next/explainer/src/18-p2-core.md deleted file mode 100644 index 51925055d..000000000 --- a/packages/chord/docs/next/explainer/src/18-p2-core.md +++ /dev/null @@ -1,324 +0,0 @@ - -# Building p2: the Core {#p2-core lede="From a list of what the app must do to a set of owners: the base code every p3 plugin builds on."} - -The notes plugin used every Chord concept. A real app is the same concepts many times over. This chapter and the -next two assemble **p3**, the Chord v2 version of p2, from the parts the guide introduced. p3 is built next to p2 in -`apps/p3`, fully isolated with its own data directory, and **without migrating p2's data** (no import of -`catalog.sqlite`, `shares.sqlite` or the subagents table). Its plugins are derived from what the app must do, not -ported one to one from p2's plugin layout. - -:::review | Review: p3 is not designed yet -The design fixes the rules (`design.md` §25) and lists the p3 decisions still open (#17, #18, #20 to #23); the -per-domain design of p3 has not started. What these three chapters show is one construction that follows the rules, -derived from the functionality map in `apps/p2/docs/p3-functions/`. Object names are working names. Where a domain -has an open decision, the text says which. -::: - -## Start from functions, not from plugins - -The functionality map in `apps/p2/docs/p3-functions/` lists what p2 does, in plain words, with four facts per -function: the data and its size, how durable it is, who may see or do it, and where it must be available. It groups -the functions into 22 domains: - -| # | Domain | # | Domain | -| --- | --- | --- | --- | -| 1 | sessions | 12 | access | -| 2 | session workers | 13 | models, credentials, settings | -| 3 | conversations and agent | 14 | prompt resources | -| 4 | shell commands | 15 | status and presence | -| 5 | chat mode | 16 | search | -| 6 | agents (subagents, one-shots, peer messages) | 17 | remote access and forwards | -| 7 | code mode | 18 | logs and notices | -| 8 | session files | 19 | dev and reload | -| 9 | environments and execution | 20 | web shell | -| 10 | workspace | 21 | TUI shell | -| 11 | terminals | 22 | transport | - -The four facts answer the four questions every Chord object needs, in order: - -:::flow wide animate | From a function in the map to code. Each fact in the map answers one question Chord needs answered. -row "The map says" -box data "data" "what, how big" -box dur "durability" "ephemeral · until restart · persisted · transcript-bound" -box who "who" "owner · guest read · guest write · worker" -box where "where" "server · worker · web · TUI · environment" -row "Chord needs" accent -box owner "the owner" "smallest object it belongs to" accent -box store "where it is kept" "Durable, persisted or in-memory document, tab" -box variant "variants" "and revocation" blue -box host "host and code" "placement; core or plugin" -arrow data -> owner -arrow dur -> store -arrow who -> variant -arrow where -> host -note "Durability decides storage: transcript-bound data is Durable's; persisted app data is a persisted document with a stable identity; until-restart data is an in-memory document or a native; tab-scoped UI state stays in the client." -::: - -**One owner per thing, the smallest object it belongs to.** A guest's grant belongs to the access owner, not to the -session; a draft belongs to one (conversation, author) pair, not to the chat plugin as a whole. Ownership nests, and -rules set at creation are written in the creating commit (Durable's `createConversation({ init })`). - -**Core or plugin.** Core owners are what the app always needs and everything builds on; they are always present, so -other code may depend on them directly. Plugin owners may be missing; anything that depends on an optional feature -does so through a registry it owns itself (chapter XIII). The risk to avoid is a **core god object**: each core owner -owns only what belongs to it. - -## The ownership map - -The result of that procedure for p3, as a starting point: - -| Domain | Owner object(s) | Host | Code | -| --- | --- | --- | --- | -| access | `Access` (grants, devices, share links), `authenticate` | server | core | -| sessions | `Catalog` (session records), lifecycle jobs | server | core | -| session workers | the supervisor (`route(Session)` + idle stop) | server | core (app code) | -| session location | `SessionLocation { session }` | worker | core | -| conversations and agent | `Harness`, `ConversationStore`, `Conversation` views | worker | core | -| status and presence | `SessionStatus` (server copy), reporters on workers | both | core | -| shell commands | `ShellJob` in the chat plugin | worker | plugin | -| chat mode | chat plugin | worker | plugin | -| agents | subagents plugin (Durable child conversations) | worker | plugin | -| code mode | code mode plugin (sandbox native) | worker | plugin | -| session files | `SessionFiles { session }` (proposed, #17) | server | core or plugin, open | -| environments | envs plugin (config, SSH pools #20) | server and worker | plugin | -| workspace | workspace plugin | worker | plugin | -| terminals | terminal host process (#23), terminals plugin | separate process | plugin | -| models, credentials | models plugin, login job + flow | server | plugin | -| settings | `Settings` (single writer, #21) | server | core | -| prompt resources | prompt plugin (watchers in the session's environment) | worker | plugin | -| search | search plugin (index on the server, feed on workers) | both | plugin | -| remote access, forwards | radius plugin, ports plugin | server | plugin | -| logs and notices | `Notices` (one channel), log capture | server | core | -| dev and reload | the reload control, `reload_plugins` tool | server | plugin | -| web and TUI shells | UI facets over the objects above | clients | shells | - -The rest of this chapter builds the core rows. Chapters XIX and XX build the plugins. - -## The base code - -```text -apps/p3/ - src/ - hosts.ts Server, Session host kinds - principal.ts P, Revocation, principalKey, revokedBy, can() - server/main.ts createHost, load, route(Session), WebSocket and Radius entry points, supervisor - worker/main.ts createHost with the start data, connect to the parent - core/ the core facets: Access, Catalog, SessionStatus, Settings, Notices (server); - SessionInfo, SessionLocation, Harness, ConversationStore (worker) - plugins/ chat, subagents, codemode, envs, workspace, terminals, models, login, - prompt, search, ports, radius, files, dev -``` - -`hosts.ts`, `principal.ts`, the `SessionInfo` start data and the `Harness` object are the ones the guide already -built for Studio (chapters I, III, IX and XIV). Base code restarts the hosts when it changes; everything in -`plugins/` reloads. - -## Who is calling: authentication - -Chord stamps principals; the app produces them. p3's server has four ways in, each turned into a principal by -`authenticate(req)` when a WebSocket connects (chapter III): - -| Way in | Principal | -| --- | --- | -| loopback without a cookie (TUI, CLI, first page load) | `{ kind: "owner", access: "local" }` | -| loopback owner browser cookie (a per-port secret file) | `{ kind: "owner", access: "local" }` | -| a device cookie (HMAC-signed, per port, 30 days), obtained with a one-time sign-in token | `{ kind: "owner", access: "device", device }` | -| a guest cookie from a share link | `{ kind: "guest", session, mode, grant, revision }` | -| the Radius relay channel | `{ kind: "owner", access: "relay" }` | - -The checks that are not Chord's stay in this function: origin checks and Fetch Metadata (a browser request from -another site gets no principal, except a top-level navigation), proxy trust, the sign-in token rules (10 minutes, -single use, at most 32 outstanding, memory only), cookie expiry and revocation lists. A failed check closes the socket -before Chord sees it. - -**One permission matrix.** Every function in the map names who may use it, and p2 checks those rules in several -places. p3 writes them once, in `principal.ts`, and every factory and `authorize` uses it: - -```ts -// src/principal.ts -type Action = "read" | "send" | "upload" | "stop" | "workspace" | "terminal" | "manage"; - -export function can(p: P, action: Action, session?: string): boolean { - if (p.kind === "owner" || p.kind === "server") return true; - if (p.kind === "worker") return action !== "manage" && p.session === session; - if (p.kind !== "guest" || p.session !== session) return false; - if (action === "terminal" || action === "manage") return false; // terminals: owner only - if (p.mode === "read") return action === "read"; // no workspace for guest read - return true; // guest write: send, upload, stop, workspace -} -``` - -Factories pick variants from it (`can(p, "send", session) ? "writer" : "reader"`), so the matrix is visible in one -file and the members a caller lacks are simply not there. - -## Access: grants, devices, share links - -`Access` is the core owner of everything that lets someone in: share links per session and mode, guest records, the -device list and the signing key. It lives on the server as the chapter IV pattern: a `local` store that persists the -documents (only stable identities may), and a `remote` view for the owner's settings screen: - -```ts -// src/core/access.ts (server, in the core server facet) -const Links = defineState("access.links", { schema: map(ShareLink), version: 1, persist: true, - initial: () => ({}), checkpointWhen: (_v, _o, info) => info.deltasSinceBase > 64 }); -const Guests = defineState("access.guests", { … }); // same shape: map(Guest), persisted -const Devices = defineState("access.devices", { … }); // map(Device), persisted - -host.serve(AccessStore, (_input, self) => { // local: may persist, acts for the host - const links = self.state(Links), guests = self.state(Guests), devices = self.state(Devices); - return { - links, guests, devices, - removeGuest: async ({ grant }, ctx) => { - await self.tx(async (tx) => { (await tx.draft(guests))[grant].blocked = true; }, ctx); - host.revoke({ grant }); // live access ends now - }, - resetLink: async ({ session, mode }, ctx) => { - const blocked = await self.tx((tx) => rotateLink(tx, links, guests, session, mode), ctx); - for (const grant of blocked) host.revoke({ grant }); - }, - signOut: async ({ device }, ctx) => { - await self.tx(async (tx) => { (await tx.draft(devices))[device].revoked = true; }, ctx); - host.revoke({ device }); - }, - … // shareLink, join, signOutAll - }; -}); - -host.serve(Access, (_input, principal, self) => { // remote view: owner only - if (!can(principal, "manage")) throw forbidden(); - const store = self.get(AccessStore, {}); - return { - variant: "owner", - guests: self.derive("guests", [store.guests], ([g]) => publicGuests(g)), // names, modes; no tokens - removeGuest: (args, ctx) => store.removeGuest(args, ctx), - resetLink: (args, ctx) => store.resetLink(args, ctx), - signOut: (args, ctx) => store.signOut(args, ctx), - }; -}); -``` - -`authenticate` is base code on the same host and checks `AccessStore`'s current documents on every new connection -(it holds a reference to the store, taken with `server.root` at start). Live access and lasting denial stay separate, as chapter VII explained: - -| Event | Lasting effect (data) | Live effect | -| --- | --- | --- | -| remove a guest | guest record blocked | `revoke({ grant })` | -| reset a share link | new token; its guests' grants blocked | `revoke({ grant })` for each | -| archive a session | grants of that session void while archived | `revoke({ session })` | -| restore a session | grants valid again; revision bumped | none: guests reconnect | -| sign out a device | device revoked until its expiry | `revoke({ device })` | -| sign out all devices | signing key rotated, tokens cleared | `revoke({ device })` for each | - -A guest who rejoins with the same browser keeps its identity; a removed guest cannot mint a new one, because the -block is in `Guests` and `authenticate` refuses it. Names are unverified, cleaned, at most 40 characters, unique per -session ("Ann (2)"). - -## The catalog - -`Catalog` owns the session records: id, title (and whether it was edited), location, model default, pinned, -archived, timestamps. It lives on the server, persisted, and it **never starts a worker**: listing sessions, showing -their status or their unread marks reads server-side copies only. - -```ts -export const Catalog = remote("p3.catalog", { - version: 1, - input: obj({}), - placement: () => Server(), - variants: { - owner: { - state: { sessions: obj({ items: arr(SessionRow) }) }, - methods: { - create: method(CreateArgs, str()), rename: method(RenameArgs), pin: method(PinArgs), - archive: method(obj({ id: str() })), restore: method(obj({ id: str() })), - }, - }, - guest: { state: { sessions: obj({ items: arr(SessionRow) }) } }, // only the guest's own session, if not archived - }, -}); -``` - -Behind it sits a persisted `local` `CatalogStore`; the guest variant is a derive over it, as in chapter IV. The lifecycle operations (archive, clone, fork, -relocate, delete) are chapter XIX; here only the rule that keeps the catalog small: **the catalog does not know which -plugins exist.** Archiving a session writes `archived: true` and revokes the session; every plugin that keeps -per-session things reconciles from `catalog.sessions` with a routine (chapter X), as the notes plugin did. The ports -plugin closes forwards of archived sessions that way, also after a crash. - -## On the worker: start data, location, harness - -Three core objects exist on every worker from its start: - -- **`SessionInfo`** (chapter IX): the start message as `constant(init)`, with the session id, data directory and the - location the server chose at start. -- **`SessionLocation { session }`** (working name), lifetime host: the current cwd and environment, with a location - revision. Relocation changes it while the worker runs (chapter XIX); natives that depend on the location follow it - with a routine and a generation (chapter IX), and conversations that follow the session's cwd are listed in the core - per-conversation policy document. -- **`Harness`** (chapter XIV): the Durable harness for the session's store, one SQLite database per session, opened - under an exclusive lock keyed by the real data directory and session id. It refuses to start when the saved root - model is not configured, and resumes interrupted Durable work on start. - -`ConversationStore` and the `Conversation` views in front of it are chapter XIX. - -## Workers: start, keep alive, stop - -Starting, supervising and stopping workers is app code (chapter I); Chord only routes and counts. p3's supervisor is -`server.route(Session, { authorize, resolve })` plus a small server object that keeps per-session worker state: - -:::steps | A worker's life, as the supervisor sees it -+++ Authorize, then start -A tab gets `Notes { S1 }`. `authorize` checks `can(p, "read", "S1")` and that the session exists and is not archived -(an archived session refuses, except an index rebuild that starts the worker without resuming work). `resolve` forks -`worker/main.ts` with the start data from the catalog, bounded by a 60 s readiness timeout. -+++ A start failure sticks -If the worker fails to start (an unconfigured environment, a missing model), the error is kept per session and -every `get` fails at once with it, until the owner presses Retry. That clears the error and the related notices. -Guests see that the session is unavailable, not the error text. -+++ Counting -The core facet on the worker reports `host.activity` (`{ references, calls, jobs, lingering }`) and the harness's -running Durable task count to the supervisor object, from a routine whenever either changes. The call carries -`ctx.incarnation`, so a report from an older worker incarnation is ignored (chapter X). -+++ Idle stop -When everything has been zero for five minutes (checked every second, as in p2), the supervisor stops the worker: -the worker closes its host (routines stop, natives close, storage flushes) and exits. Durable work stays persisted -and resumes on the next start. Detached work that must keep the worker alive (a shell command whose caller left) is -a job, so it counts. -+++ Crash -If the worker exits on its own, the link drops. The supervisor records the exit code and a redacted stderr tail -(at most 4,000 characters) for that incarnation and posts one notice. References that still exist make Chord call -`resolve` again with backoff; a passive viewer recovers without a click. -+++ Shutdown -On server shutdown the supervisor stops every worker, with a forced exit after three seconds. Durable work remains -persisted. -::: - -## Status without a worker - -The session list shows per session whether it is running, busy (which conversations), its task count, usage and cost, -and an error. None of that may start a worker. The pattern is chapter X's pub/sub: a reporter on each worker (a -routine over the harness's state) calls `SessionStatus.report(summary)` on the server; `SessionStatus` keeps the latest -value per session and ignores calls whose `ctx.incarnation` is older than the newest it has seen; clients read -`SessionStatus`, which lives on the server. Children lists, unread marks, search index rows and artifact lists use the -same pattern (chapter XIX). - -## Settings and notices - -**Settings** (default model and thinking, run settings, compaction, queue modes, code mode switch, owner name) have -**one writer**: a core `Settings` object on the server owns the file, validates writes and serializes them; every -other process follows its state through a reference instead of watching and rewriting the file itself. p2 watches -settings, models and auth in every process and writes them from two, which is decision #21. - -**Notices** are one channel: one core `Notices` object on the server, written by everything that fails or finishes -(worker crashes, reload results, model errors, login outcomes, grants), keeping one latest notice per source, at most -50. The owner's shell reads one list; dismissing is a method. p2 has five notice channels for the same thing. - -:::review | Review: settings and file formats (#21, #22) -Single writer for settings, models and auth (#21) and versioning on-disk formats such as `envs.json` like documents -(#22) are open p3 decisions in `design.md` §26. The construction above assumes "one owner on the server, others -follow its state" for #21. -::: - -:::note | Built so far -Nothing of p3 exists. The functionality map is complete (`apps/p2/docs/p3-functions/`, five area files plus the -condensed `README.md`), and the design's app rules (`design.md` §25) are decided. p3 starts after the layered rebuild -and the playground's first real proof. -::: diff --git a/packages/chord/docs/next/explainer/src/19-p2-sessions.md b/packages/chord/docs/next/explainer/src/19-p2-sessions.md deleted file mode 100644 index 9d40c2287..000000000 --- a/packages/chord/docs/next/explainer/src/19-p2-sessions.md +++ /dev/null @@ -1,276 +0,0 @@ - -# Sessions and Conversations {#p2-sessions lede="The session lifecycle, status without a worker, chat, shell commands, subagents and search, each mapped to an owner and the Chord parts it uses."} - -This chapter builds the domains a person touches most: the session list and its operations, the conversation view -and composer, shell commands, chat mode, subagents, messages between sessions, search and unread marks. Each section -names the owner, the host, and the Chord parts it uses; code appears only where it shows something new. - -## The session lifecycle - -The catalog (chapter XVIII) owns the session records. Its operations, by the Chord part each needs: - -| Operation | Owner and form | Notes | -| --- | --- | --- | -| create | `Catalog.create`, a method | absolute directory, `~` expanded, must be a directory; on an environment, resolved there through the envs plugin; refuses an unknown model; model and thinking default to the last used | -| rename, pin | methods on `Catalog` | title 1 to 120 characters, whitespace collapsed; an explicit title is never replaced automatically | -| automatic title | a routine on the search feed | the first user message, line breaks to spaces, 100 characters | -| agent rename | an agent tool on the worker calling `Catalog` | root conversation only | -| archive, restore | methods; revocation; reconciling routines | below | -| clone, fork, relocate | jobs on the catalog store | outlive the call, cancellable, one outcome | -| delete | a job; decision #18 | below | - -### Archive and restore - -Archiving must stop the worker whatever it is doing, end guest access, and close the session's port forwards, while -keeping its storage. The catalog does none of the plugin-specific parts: - -:::steps | Archiving S1 -+++ One write -`Catalog.archive({ id: "S1" })` sets `archived: true` in the session record, in one transaction. -+++ Revoke the session -The catalog calls `host.revoke({ session: "S1" })`. Every guest principal of S1 has `{ session: "S1" }` in its -`revokedBy`, so their objects end and their links close on the server and on the worker (chapter VII). New -connections are refused by `authenticate`, because grants of an archived session are void. -+++ Stop the worker -The supervisor reconciles from `catalog.sessions` with a routine: an archived session's worker is stopped, and -`authorize` refuses to start it again. -+++ Plugins reconcile -The ports plugin's routine on `catalog.sessions` closes forwards of archived sessions. The search plugin marks the -session's rows archived. If the server crashed between the write and these steps, the routines run again with the -current value at the next start: nothing is a notification that can be lost. -+++ Restore -`Catalog.restore` clears the flag. Grants are valid again, so guests reconnect and authenticate as before; forwards -stay closed. -::: - -### Clone, fork, relocate - -Each is work that outlives its call, can be cancelled and has one outcome, so each is a **job** (chapter XI) on the -catalog's local store, keyed by the owner's request id. They differ in what they pause: - -- **Clone** copies the session's store, uploads, downloads and artifacts under a new id, with the source worker - paused (refused while work runs) and title "(copy)"; directories `0700`, files `0600`, no links, no overwrite; a - partial copy is removed on failure. -- **Fork** before a user message copies the active context as of the entry before it into a new session (title - "(fork)"), with the message waiting in the new session's editor. It needs two workers: the source reads history, - the new one is seeded empty. A failure archives the copy. -- **Relocate** (cwd, environment or both): refused while the session is busy (naming the running subagents) or when - a subagent has its own cwd and the move crosses machines. The job fences the worker (new admissions wait), updates - the record, and moves the running worker in place by updating `SessionLocation`: natives that follow the location - switch with a generation (chapter IX), and conversations that follow the session's cwd are rewritten in one Durable - commit before work resumes. After 60 s it restarts the worker instead; on failure it releases the fence. - -:::review | Review: jobs as objects (#9) -The design defers whether a long relocation or a login is a record inside its owner (watched through `running`) or an -object of its own (`Relocation { id }`). The construction above uses records. Decide when login is ported -(`design.md` §15, §26). -::: - -### Delete - -p2 has no session delete. p3 needs one that removes the session everywhere: every document tagged with the session -(`store.retireByTag(SessionTag, id)`, chapter V), the worker's databases (the Durable store), the session's files, the -search rows, read marks, children copies, forwards, guest records, and finally the catalog entry. The design commits to -this outcome (`design.md` §25); how plugins take part is open. - -:::review | Review: session delete (#18) -Options in `design.md` §26: per-session objects plus `SessionTag` plus one session directory (delete is retire by tag -and remove a directory, plugins need do nothing), or owner hooks contributed through a catalog registry. Also open: -whether the search index is per session (deleted with it) or global with a hook. Who holds the store for -`retireByTag` is not specified either. -::: - -## Status, children, unread: reports to the server - -The session list must show status, subagents and unread marks for sessions whose workers are not running, and must -not start them. All three follow chapter X's pub/sub pattern: a routine on the worker calls a server object with the -current summary, the server object keeps the latest value per session (persisted where it must survive a restart), -ignores calls from an older `ctx.incarnation`, and clients read the server object. - -| Summary | Source on the worker | Kept on the server | -| --- | --- | --- | -| status: running, error, task count, busy conversations, usage and cost | the harness's task graph and usage | `SessionStatus`, in memory: a stopped worker is "not running" | -| children: name, model, thinking, tools, cwd, last entry, archived | the subagents plugin's child list | `Children`, persisted | -| latest root answer | the search feed | `Unread`: compared with the owner's read mark, persisted, shared by all owner devices | - -Guests see their own session's status without the error text. The owner's read mark is written when the root -conversation is visible, and a new session starts as read. - -## Conversations - -The harness (chapter XIV) owns the Durable store. The UI needs, per conversation: a live window of entries, older -pages, usage numbers, the working indicator, the inbox (queued inputs), and the actions. Two core objects provide them: - -- **`ConversationStore { conversation }`** (local, on the worker): opens the Durable watch as its native, reaching - the harness through the core-private `HarnessRuntime` registry entry, and exposes a **window** as a derive: the - newest entries (at most 100 or 512 KiB, at least 20) plus the conversation's documents. The window's start stays - put while it is within 1.5 times the limits, so new entries arrive as small appends instead of a shifted list - (chapter VIII's window pattern). -- **`Conversation { session, conversation }`** (remote, placement `Session`): the per-principal view. Variants by - `can()`: `owner` (everything, including model, thinking and handoff), `writer` (guest write: send, steer, stop, - compact, shell), `reader` (the window only). - -```ts -host.serve(Conversation, ({ session, conversation }, principal, self) => { - const store = self.get(ConversationStore, { conversation }); - const policy = self.get(ConversationPolicy, { conversation }); // core: readOnly, followsSessionCwd - const author = authorOf(principal); // structured: kind, id, display name - if (!can(principal, "send", session)) return { variant: "reader", view: store.window, page: store.page }; - return { - variant: can(principal, "manage") ? "owner" : "writer", - view: store.window, - page: store.page, // older history: returns data - send: ({ text, attachments, mode, requestId }, ctx) => { - if (policy.value.readOnly) throw forbidden("read-only conversation"); - return store.submit({ type: "input", content: text, attachments, author, whenBusy: mode, requestId }, ctx); - }, - stop: (_args, ctx) => store.stop(author, ctx), // allowed even when read-only - … - }; -}); -``` - -What each function becomes: - -| Function | How | -| --- | --- | -| send, retried send | Durable `submit` with the author's `requestId`: a retry with the same id is admitted once (Durable dedupes, chapter XIV) | -| steer or queue while busy | `whenBusy: "steer"` or a follow-up; the inbox is `docs["pi.inbox"]` in the window | -| attribution | the view derives the author from the principal; the entry stores it structured, the model sees a speaker block; nothing parses text | -| wait for prompt resources | the first input of a fresh worker waits until the prompt plugin's resources are loaded (a registry readable the store follows) | -| stop, dequeue | withdraw queued inputs, abort the run; the caller's texts return to its editor, other authors' to their **restored drafts** | -| restored drafts | a keyed document per (conversation, author), tagged `SessionTag`, persisted in the store; claimed once, first tab wins | -| read-only conversations | the core per-conversation policy document, written at `createConversation({ init })` by whoever creates the conversation; send, dequeue, configure, compact and shell refuse; stop is allowed | -| history pages | a method returning a page (100 entries, 1 MB, 2,000 scanned); compaction and reset markers always included | -| model and thinking | owner variant only; validated against the models plugin's catalog; the root conversation's model is also reported as the session default | -| compaction, handoff | Durable operations through the store | - -**Drafts in the composer** are UI state, not Chord state: the web shell keeps one draft in `sessionStorage`, the TUI -keeps one per conversation across UI replacements, and pending sends keep their request id until acknowledged -("Sending when connected"). Only restored drafts, which the worker produces, are documents. - -## Shell commands - -`!cmd` runs a command in the conversation's cwd on its environment and records the result as an entry; `!!cmd` does -the same but keeps the output out of the model's context. One command at a time per conversation; the live tail is -at most 50 KB or 2,000 lines, sampled every 200 ms; the full output spills to a file; the result entry is placed at the -run's next boundary while the agent is busy and never starts a run. This is the design's job example almost as is: - -```ts -const ShellJob = defineJob("chat.shell", { - version: 1, - args: obj({ command: str({ max: 100_000 }), include: bool(), author: Author }), - progress: obj({ output: str() }), - result: ShellResult, - concurrency: 1, // one per conversation -}); - -const results = self.state(ShellResults); // keyed by run id: dedupe after completion -const shell = self.job(ShellJob, { - run: async ({ command, include, author }, ctx) => { - const out = new OutputBuffer({ maxBytes: 50_000, maxLines: 2000, retain: "tail" }); - const env = await environmentFor(ctx); // the conversation's environment, chapter XX - const r = await env.exec(command, { cwd: env.cwd, signal: ctx.signal, spill: spillPath(ctx.jobId) }, - (chunk) => { out.push(chunk); ctx.progress({ output: out.snapshot().text }); }); - const data = { command, output: out.snapshot().text, exit: r.code, cancelled: ctx.signal.aborted }; - await conversation.submit( - { type: "write", entry: shellEntry(data, author, include), requestId: `shell:${ctx.jobId}` }, ctx); - return data; - }, - finish: async (done, tx) => { - (await tx.draft(results, done.id)).outcome = done.kind === "done" ? done.result : { error: done.error }; - }, -}); -``` - -The request id `shell:${ctx.jobId}` makes the entry idempotent: an attempt interrupted after the write cannot write -it twice. Cancel is `shell.abort(id)`, allowed for the command's author and the owner. A retried `runShell` with the -same request id returns the running job (dedupe of unsettled runs), or the stored result (the owner's own dedupe after -completion, chapter XI). p2's "cancel when nobody watched for 30 s" is dropped: a job keeps its owner alive and does -not depend on viewers. - -## Chat mode - -Chat mode is a per-conversation switch that everyone in the session sees and only the owner changes; forks and new -subagents start with it off. Because it must fork and rewind with the conversation, it is **transcript-bound**: a -Durable conversation document, not a Chord document. Messages without `@pi` become display-only Durable `write` -entries (author, text, attachments, reply quote); a message with `@pi` becomes an input with a digest of the chat -since the last mention (at most 100 messages, 20 files, 4 images), recording which entries it passed on. The reply -quote is derived on the worker from the entry id the client names: the client never supplies author or quote text. - -## Subagents and one-shot agents - -Subagents are Durable child conversations owned by tasks, as in Durable's own background-subagent example: one -`subagent` tool spawns, steers, waits for, stops and lists them; each child is owned by a background anchor task, and -each message is delivered by a reporter task that posts the answer back to the parent once, with request ids so a -restart sends nothing twice. Chord adds only three things: - -- the **child list** for the sidebar is reported to the server (`Children`, above), so it shows without a worker; -- a child that follows the session's cwd, or a read-only one-shot agent created by a code-mode script, is marked in - the core **per-conversation policy document** when it is created, so chat and relocation enforce it without knowing - the subagents plugin (`design.md` §25 porting guideline); -- archive and restore of a child from the UI are methods on the subagents plugin's view, refused for one-shots. - -Subagents cannot spawn subagents; thinking levels are clamped to what the model supports, in one helper shared with -the session's model choice. - -## Messages between sessions - -The root agent can list other sessions and send one an attributed message. The tool runs on the sender's worker and -calls a server object through `chord(fn, context)` (chapter XIV): - -```ts -const peerSend = defineTool({ - name: "session_send", - execute: (args, api, context) => chord(async (ctx) => { - const peers = await ctx.get(Peers, {}); // server, worker variant - await peers.send({ to: args.session, text: args.text, mode: args.mode, requestId: `peer:${api.taskId}` }, ctx); - return { output: [{ type: "text", text: "sent" }] }; - }, context), -}); -``` - -`Peers` checks that the sender is a root conversation of a non-archived session, refuses sending to itself and -archived recipients, then gets the recipient's `Conversation` (which starts its worker) and submits with the -sender as a structured author and the same request id (the tool task's id, stable across restarts), so a retried -tool call delivers once. The sender's -incarnation is `ctx.incarnation`; a call from a replaced worker is refused. Only owner-initiated turns may use the -tool; guests never. - -## Search - -The search plugin indexes user and assistant messages of root and subagent conversations (no shell output, no -Durable internal inputs) for title, cwd and message search across sessions, active or archived. - -:::flow wide animate | The search feed. The worker pushes pages from a cursor; the server owns the index and never starts a worker to answer a query. -row "Worker S1" blue -box commit "Durable commit" "in any conversation" -box wake "wake" "a routine on the harness's commit signal" -box page "read from cursor" "200 entries per page" -row "Server" accent -box index "SearchIndex" "native: SQLite FTS" accent -box cursor "cursor per conversation" "moves after the whole range" -box query "query" "terms, scope, at most 41 hits" -arrow commit -> wake -arrow wake -> page -arrow page -> index "report(page)" -arrow index -> cursor -note "Reports carry ctx.incarnation. Indexing is idempotent: a page sent twice changes nothing." -::: - -The index is a native owned by a host-lifetime object on the server; cursors live with the index, so after a crash -the worker asks for the cursor and continues. Rebuilding one session's index drops its rows and rescans, starting -an archived session's worker without resuming its work. Queries are methods returning data: at most 41 hits, -snippets of 300 characters, queries up to 500 characters. The same feed sets automatic titles, moves "updated at" -forward and supplies the latest root answer for unread marks. - -## Presence - -Viewers per session are `host.references(Conversation)` on the worker, counted per client (chapter VIII), and -reported with the status. Connected guests (name and mode) are the references whose principal is a guest. Presence -is not keep-alive: the supervisor uses `host.activity`, which also counts calls and jobs. - -:::note | Built so far -None of this exists in p3. The parts it relies on are Durable (built: submit with request ids, steer and queue, -child conversations, background tasks) and the Chord layers L3 to L9 (not built). -::: diff --git a/packages/chord/docs/next/explainer/src/20-p2-machines.md b/packages/chord/docs/next/explainer/src/20-p2-machines.md deleted file mode 100644 index 51b9e7546..000000000 --- a/packages/chord/docs/next/explainer/src/20-p2-machines.md +++ /dev/null @@ -1,245 +0,0 @@ - -# Machines, Files, Models {#p2-machines lede="Environments, workspace, terminals, session files, ports and Radius, models and logins, prompt resources, logs, reload and the shells: the rest of p3."} - -The last domains are the ones that touch the outside world: other machines over SSH, the file system and git, -terminals, the relay, model providers and their credentials. Most of them are native objects, and most of them are -places where p2 has the same thing several times (four SSH connections to one environment, five notice channels, -three copies of the environment config parser). The rule that fixes that is the one from chapter XVIII: one owner per -thing. - -## Environments - -The **envs plugin** owns the environment configuration: name, host, user, port, identity file, shell and its -variables, plus a managed ed25519 key per environment and its own `known_hosts` (kept apart from the user's, aliased -per environment name). - -| Function | Owner and form | -| --- | --- | -| list, add, edit, delete (refused while a session, archived too, uses it) | `Envs` on the server: a persisted document behind an owner-only view | -| validation (no leading `-`, no whitespace or control characters in host and user) | the view's methods, before the write | -| a hand-edited `envs.json` is picked up; an invalid file keeps the last valid set and shows an error | a native file watcher (poll fallback) in the envs store; one writer (#21) | -| environment revision: a hash of the config, part of its identity | a derive | -| connection test (connect, deploy the daemon, platform, latency, warnings) | a job; the result per revision in a document | -| unknown or changed host key: fingerprints, trust after confirmation, forget | a flow (chapter XII) on the server | -| browse directories (`~`, relative paths, at most 10k scanned, 1,000 shown) | a method with a **call-scoped** SSH session (chapter IX) or the pooled connection | -| a changed environment restarts its sessions' workers once idle | a routine on the envs store and the status reports | - -**Execution in the agent.** The worker's harness asks `HarnessEnvironments` for an environment per target (chapter -XIV). The envs plugin contributes it from its worker facet: local execution at the session's cwd, or a remote one over -SSH through the `pi-env` daemon, deployed per platform and connecting at the first operation. A remote session never -falls back to local tools. The environment follows `SessionLocation`: after a move, the old connection closes once -the move has committed and watchers reopen at the new location. - -```ts -// envs/worker.ts -export default defineFacet((host, self) => { - const location = self.get(SessionLocation, { session }); - const pool = self.get(EnvPool, {}); // one owner of SSH connections (#20) - self.contribute(HarnessEnvironments, (target) => - target.env === "local" ? localEnv(target.cwd) : pool.environment(target.env, target.cwd)); -}); -``` - -:::review | Review: connection pools (#20) -SSH connections to one environment exist four times in p2 (worker environment, browse connection, test connection, -terminal host). `design.md` §26 asks for one owner with a pool or deliberate per-use connections. `EnvPool` above is -the first option: a native object that closes idle connections after 10 minutes and invalidates them on a config or -key change. Reloading the plugin that owns the pool kills running remote commands (chapter XV), so it should change -rarely. -::: - -## Workspace and file index - -The **workspace plugin** serves the viewed conversation's working tree (a subagent's own cwd included) on the worker: -list a folder (at most 2,000 entries, `.git` hidden), read a file (1 MiB, binary detected), git status, diff, log and -single commits, literal text search, and all paths for name filtering. Every query is a **method returning data**, not -state: results are bounded and requested by the UI. Only change notifications are state: a counter in a document, -bumped by a native watcher of the tree, `HEAD`, the index and config (debounced 100 ms); viewers reload what they show -when it changes. - -Two rules live in one place each instead of three: - -- **Confinement:** path rules, a canonical root, no `.git`, the app's private data directory denied (also excluded - from search). One helper used by listing, reading, uploads and the file index. -- **Git hardening:** no optional locks, no fsmonitor, filters disabled, no prompts or pager, no system config, 10 s - timeout, output caps. - -Downloads (25 MiB, byte ranges for media) go through the HTTP component (chapter XIII) with a route on the server that -reads chunks from the worker; uploads into the project use the chunked protocol of chapter XIII (1 MiB chunks, temp -file then rename, never overwrite, writes serialized). Guest read gets no workspace: the view has no variant for it. - -The **file index** for `@` completion belongs to the same owner, so completion consumes the index instead of scanning -on its own: built from the environment's listing (else git or a walk, at most 50k paths), rebuilt at most once per -second while files change, discarded after a move. Review comments drafted on files and diffs are browser state until -sent; sending them is an ordinary `Conversation.send`. - -## Terminals - -Terminals must survive server and worker restarts, so their PTYs live in a **detached terminal host process**, one per -data directory, with its own process lock; it exits after 10 minutes with no terminals and no clients. The terminals -plugin on the server is the owner the UI talks to: list (title, cwd, environment, size, exit status), open at the -session's cwd and environment (refused for archived sessions), attach, close (asking first if a foreground job runs). -Terminals are **owner only**. - -Bytes do not go over object links. Attaching opens a **channel** (chapter XIII) with a single-use token (30 s): JSON -control on the object, bytes on the channel, with its own acknowledged window (256 KiB, acknowledgements per 64 KiB; a -viewer that falls behind gets a fresh screen snapshot; an 8 MiB backlog disconnects). Attaching gets the current -screen from a headless emulator with 10k lines of scrollback. - -:::review | Review: the terminal host process (#23) -The terminal host runs its own code. Is it part of the revision, reloaded or restarted with it (and running terminals -end), or deliberately outside it with a "host runs older code" notice as in p2? Open in `design.md` §26. -::: - -## Session files - -Attachments (25 MiB, at most 20 per message), sent-file snapshots made by the agent's `send_file` tool, versioned -artifacts (HTML, SVG, Markdown, text; 2 MiB; each update a new immutable version) and shell spill files are all -per-session bytes with caps, immutable ids and range reads; p2 has two upload routes, two caps and two directory -layouts for them. The proposal is one owner: - -```ts -export const SessionFiles = remote("p3.session-files", { - version: 1, - input: obj({ session: str() }), - placement: () => Server(), // readable without a worker - variants: { writer: { … }, reader: { … }, worker: { … } }, -}); -``` - -`SessionFiles { session }` on the server holds the index (per kind, readable without a worker), the chunked upload -and read protocol, the caps per kind, copying on clone and fork, and deletion with the session. Attachments are copied -into a remote environment before an input is admitted, with no fallback on failure. Artifact previews run in a -sandboxed frame with its own CSP and no app identity; ordinary file previews allow only safe types. - -:::review | Review: session files (#17) -Proposed in `design.md` §26, not decided. Open sub-points: one preview limit (p2 has 256 KiB for sent files and -512 KiB for artifacts), what a fork copies of the artifact list, and how attachments are copied into remote -environments. Related: #22, whether on-disk layouts such as downloads and uploads get versions like documents. -::: - -## Ports and Radius - -The **radius plugin** keeps the outbound relay connection, a native in a host-lifetime object on the server: sign-in -by device code from the CLI (credential in a private file), token refresh, a heartbeat in the link adapter (20 s ping, -60 s timeout; TCP does not notice a sleeping laptop), backoff from 1 to 30 s, a disable switch. The relay delivers two -kinds of channels: - -```ts -// radius/server.ts (the entry point: one line, as in chapter I) -radius.onChannel((ws) => host.accept(webSocketLink(ws), { principal: { kind: "owner", access: "relay" } })); -radius.onForward((channel) => proxyToForward(channel, host.registry(RadiusForwards).value)); -``` - -Object links become owner principals with `access: "relay"`: owner, but treated as remote (no localhost-only -powers). p2 had to fake a loopback request with a secret header for this; with `accept` that is gone. - -Forward channels go to `localhost:` (Host, Origin and Location rewritten; 502 when nothing listens; anonymous -visitors only to public forwards). Which forwards exist is not radius's business: it owns a registry, -`radius.forwards`, and the **ports plugin** contributes its forwards (at most 50, random ids, a public toggle that -needs confirmation, an agent tool `ports` for top-level owner-initiated conversations). Without the ports plugin the -table is empty and radius still works: an optional dependency is a registry owned by the dependent (chapter XIII). The -ports plugin closes forwards of archived sessions with a routine on `catalog.sessions` (chapter XIX). - -:::review | Review: may a plugin call `accept`? -`design.md` §3 shows the Radius line in the server's start code and calls entry points "app or adapter code". If -radius is a plugin, its facet needs the host's `accept`, which no other part of this guide gives a plugin. The -alternative is that the start code keeps the entry point and gets channels from the radius object. Not specified. -::: - -## Models and credentials - -The **models plugin** owns the model runtime (pi-ai's models with custom providers and the model cache) in a -host-lifetime object, and the credentials. It contributes the runtime to `HarnessModels` (chapter XIV), so after a -models reload the next request uses the new runtime while a running stream finishes on the old one. **Credentials -never go to clients**: the views expose providers, login methods and which type is signed in, never tokens. Every -secret is registered for redaction in one place, the log capture. - -Views: a model catalog for pickers (names, ids, context windows, thinking levels; at most 300 matches in the web), -logout (refused during a login), refresh a provider's model list. The default model and the cycling scope are -settings (chapter XVIII). - -### Login - -A login is the guide's flow example (chapter XII): a **job** on the server that runs pi-ai's login, and a flow that -shows its steps to the starting owner. - -```ts -const LoginJob = defineJob("models.login", { - version: 1, args: obj({ provider: str(), type: AuthType }), progress: LoginState, result: obj({ ok: bool() }), - concurrency: 1, // one login at a time on the server -}); - -const login = self.job(LoginJob, { - run: async ({ provider, type }, ctx) => { - await runtime.value.login(provider, type, { - signal: ctx.signal, // 10 minute deadline, cancel, leaving - prompt: (prompt) => steps.ask(prompt, ctx), // text, secret, select, manual_code - notify: (event) => steps.notice(event, ctx), // info, auth_url, device_code, progress - }); - return { ok: true }; - }, - finish: async (done, tx) => { /* record the outcome; refresh the model list */ }, -}); -``` - -`prompt` is a promise-based callback inside the work; the resolver stays in the job (`steps.ask` records the step in -the flow state and waits for `answer`), and nothing crosses an object boundary. On a relayed connection the device -code method is selected automatically. The flow is visible only to the owner principal that started it; others are -refused. Leaving ends it: when the starting owner's references are all gone (presence, not an unmount), a routine -aborts the job. Codes, URLs and typed secrets are redacted from logs. - -## Prompt resources - -`AGENTS.md`, skills and templates are read in the session's environment, watched (poll fallback, last valid state), -and expanded on the worker before sending. The **prompt plugin** owns them per session on the worker, contributes the -system prompt section and skills to `HarnessExtensions` as readable contributions (a change recomposes), and exposes a -readiness readable: the first input of a fresh worker waits until it is ready (chapter XIX). After a move the watchers -reopen at the new location. - -## Code mode - -Code mode is a plugin (chapter XIV): its `pi-codemode` sandbox is a native; scripts call tools as nested Durable calls -and replay after a restart; the tool entry looks up the current sandbox when it starts. A reload seals and drains the -old sandbox for at most 30 s. The on/off switch is a setting. - -## Logs and notices - -**Log capture** tees stdout and stderr of the server, workers and detached hosts without changing their behaviour, -redacts known secrets and credential patterns, and keeps bounded rings (at most 128 sources, 1,000 lines or 256 KiB -each). Detached hosts publish redacted snapshots to a private file. Logs are read without waking a stopped worker. -**Notices** are the one channel from chapter XVIII; dismissal stores only notice ids. - -## Dev and reload - -The dev plugin is chapter XV's reload control for p3: a source watcher marks plugins dirty and never applies anything; -the `reload_plugins` agent tool and a control in the owner's shell reload the changed plugins, or restart all hosts for -base code (through the supervisor, reopening the sessions that were running). UI bundles are published only after a -successful backend change; a web page detects a new manifest and reloads with its drafts and navigation state saved; -the TUI swaps its sandboxed UI code with a snapshot of its state. A local Unix socket (`0700`) accepts development -commands without a network admin port. - -## The shells - -The web and TUI shells are UI facets over the objects above. Two rules matter for Chord: - -- **UI state survives navigation, reload and reconnect on purpose.** Composer drafts, pending sends with their request - ids, reading positions and review drafts are kept in tab storage or the TUI's replacement snapshot. A reconnect - re-holds and gets snapshots; nothing is lost that was not in flight. -- **Leaving is never derived from an unmount.** A plugin reload remounts components (chapter XV). Cancelling a login, - closing an overlay or giving up a lease is an explicit action or a dropped reference. - -The TUI launcher keeps downloaded UI code in a permission-dropped worker with no access to files, network or -credentials; local side effects (copy, upload, download, open a URL, an external editor) need a real user gesture or a -confirmation. That is app code around Chord, not part of it. - -## Transport - -The last domain, transport, is chapter XVII: one message format, per-part limits (16 MiB compressed and 64 MiB decoded -on client WebSockets, instead of p2's 1 MiB input cap and 32 pending calls), one byte window per link, re-holds with -snapshots after a reconnect, and a protocol version that makes an outdated web page reload. - -:::note | Built so far -None of p3 exists. pi-ai's login API, pi-codemode and the `pi-env` daemon exist and are used by p2 today; the HTTP -component and channels are deferred until after the kernel's first real proof (chapter XIII). -::: diff --git a/packages/chord/docs/next/explainer/src/20a-compared.md b/packages/chord/docs/next/explainer/src/20a-compared.md deleted file mode 100644 index aa4fcd2c2..000000000 --- a/packages/chord/docs/next/explainer/src/20a-compared.md +++ /dev/null @@ -1,146 +0,0 @@ - -# Other Ways to Do It {#compared lede="Cordis and DeepSeek Harness attack the same problem, plugins that come and go while an agent runs, inside one process. What that buys, and what it costs."} - -Chord v2 is not the only answer to "replace plugin code while the app keeps running". The closest relative is -**Cordis**, a TypeScript plugin framework by Shigma that has been the plugin kernel of the Koishi chatbot framework -since 2019, and its most visible user, **DeepSeek Harness** (`dsh`), DeepSeek's open-source agent harness, whose -motto is "everything is a plugin". This chapter compares them with Chord v2 on the questions the guide cared about: -who owns side effects, how plugins depend on each other, what a reload keeps, what a crash keeps, and how far the model -reaches beyond one process. - -:::note | Sources and status -DeepSeek Harness at commit `687f282` (release 0.2.1-rc.2), which vendors Cordis as `@deepseek-ai/cordis` 4.0.5 (Cordis -RC7 with local changes, `vendor/README.md`): its Cordis tutorial (`docs/cordis-tutorial/`), `docs/architecture.md`, -`docs/subsystems/persistence.md`, `docs/subsystems/core.md` and `packages/boot/hmr/README.md`; the Cordis paper "A -Programming Paradigm for Spatiotemporal Composability" (arXiv 2608.25512). Cordis and dsh are built and in use; Chord v2 -is the design this guide describes. The comparison is between a working system and a design. -::: - -## Cordis in one example - -A Cordis plugin is a function (or an object with `apply`, or a `Service` subclass) that receives a **context**. It may -declare the services it needs with `inject`, and everything it does to the world goes through the context: - -```ts -export const name = "greeter"; -export const inject = ["tools"]; // activate only while a `tools` service exists - -export function apply(ctx: Context) { - ctx.tools.register(defineTool({ name: "greet", /* … */ })); // a registration is an effect: undone on unload - ctx.on("session/start", (s) => log(s.id)); // an effect-bound listener - ctx.effect(() => { // any other side effect, with its disposer - const timer = setInterval(work, 200); - return () => clearInterval(timer); - }); -} -``` - -Each plugin instance is a **fiber** with states `PENDING → LOADING → ACTIVE → UNLOADING → DISPOSED` (or `FAILED`). -Every effect, registration, listener, child plugin and provided service is recorded on the fiber, and unloading runs -the disposers. The paper calls this **temporal composability**: a component can be removed at any time and reverses its -own effects. Its counterpart, **spatial composability**, is the dependency side: `inject` is live, so a plugin starts -when its services appear, and unloads (and later reloads) when a service it depends on goes away or is replaced. The -paper's claim is that independently written components can therefore be combined and replaced without disturbing each -other. The next sections check how far that holds. - -**Services** are named capabilities on the context (`ctx.provide("x", value)` or `class X extends Service`). Names are -runtime strings in one flat namespace per application; TypeScript declaration merging makes `ctx.x` typecheck, but -nothing checks the service's shape at runtime, and optional dependencies are probed with `ctx.get("x")`. Scopes come -from the context tree: `ctx.extend()` derives a child context, `ctx.isolate(name)` gives one subtree its own binding of -a service, so two parts of an app can use two different `shell` providers. - -**Hot reload** follows from the two halves: the reload plugin unloads the old fiber (unwinding all its effects) and -loads the new module with the same config. Nothing held in the old instance's closures or fields survives; a plugin -that needs state across reloads must keep it in a service or on disk. Config edits are diffed by entry id and -mount, unmount or reconfigure only what changed; config fields declared volatile update in place without a remount. - -## DeepSeek Harness on Cordis - -dsh applies "everything is a plugin" to an agent: the model adapters (`ctx.llm`), the tool registry (`ctx.tools`), the -session log (`ctx.sessions`), the agents (`ctx.agents`) and the agent loop itself (`ctx.agentLoop`) are plugins and -services. A running dsh is a plugin tree composed at boot from a **profile** (`web`, `headless`, `sdk`, `acp`) that -stacks **bundles** of Cordis config rows, with user patches on top. There is no privileged core: replacing the agent -loop is a config patch. - -| Concern | How dsh does it | -| --- | --- | -| sessions | a durable, append-only event log per session (JSONL backend, single writer, `flush` as the durability barrier); model history is derived from the events | -| crash mid-turn | the log keeps everything appended; on resume the agent loop closes the open turn by appending missing tool errors and a synthetic `turn/end { reason: "interrupted" }` | -| tools | `ctx.tools.register(defineTool(…))`, an effect of the registering plugin; calls go through pre/execute/post policy waterfalls | -| approval | closed by default; `allowed-once` is the only grant; plugin installation needs full access or per-call approval | -| reload | a source watcher evicts changed modules; Loader entries running affected plugins stop and start again with the new code; a failed replacement is an activation failure, its previous code no longer runs | -| processes | one plugin tree in the Host process; the browser client and Electron's renderer talk to it over RPC streams | -| distribution | a Plugin Manager installs and enables bundles per profile; plugin markets are optional indexes | -| subagents | local subagents with their own durable sessions, and external ones | - -## What the guarantees are in practice - -Reading the Cordis source vendored in dsh and dsh's own documentation, the guarantees are narrower than the paper's -framing. An earlier review for p2 came to the same result (`apps/p2/docs/p3-reviews/dsh-compare.md`). - -- **Reversal is cooperative and unordered.** `_unload` runs all of a fiber's top-level disposers concurrently - (`Promise.all` in `vendor/cordis/src/fiber.ts`); order holds only inside one effect, and the primer tells authors to - "keep the related work in one effect" when teardown order matters. A disposer that throws is logged and skipped. A - disposer that never settles stalls the teardown; dsh's jobs service works around this by force-failing a job whose - cancel throws and warning that "the work may be orphaned" (`packages/jobs/jobs-local/README.md`). Reversal is as - good as every disposer, and nothing checks that a plugin registered all its effects. -- **Behaviour depends on registration order, and a reload changes it.** `emit`, `waterfall`, `serial` and `bail` - listeners run in registration order (`docs/cordis-primer.md`). Registering appends the listener (`push` in - `vendor/cordis/src/events.ts`, unless `prepend`), so when a plugin reloads, its listeners move to the end of the - list. A tool-policy listener that ran first runs last after its own plugin reloads; a waterfall that short-circuits - now sees a different decision first. -- **Dependencies are strings, and every provider reload restarts its dependents.** A fiber's activation key is the - concatenation of its injected providers' fiber ids (`_refresh` in `fiber.ts`), so replacing a provider unloads and - reloads every plugin that injects it, with their in-memory state. Service APIs are "pre-stable; update every - consumer" (`AGENTS.md`): there is no versioned contract between a provider and its consumers. -- **Reloading a core service ends live work.** Unloading the agent provider "stops and drains every live handle it - made" (`packages/core/agent/README.md`), so running agents stop; reloading the session persistence backend under - live sessions invalidates their write handles, and "writes fail loudly until the sessions restart" - (`.agents/notes/implemented/architecture/2026-08-27-handle-based-session-persistence.md`). -- **Registrations leave a gap.** A reloading plugin's tools and other registry entries are removed with its old fiber - and added again by the new one; anything that looks them up in between does not find them. - -None of this is a bug in the narrow sense: each point is documented, and dsh is built around it. But "components -interleave without disturbing each other" holds only for plugins that own nothing other plugins depend on, register -every effect, order nothing by registration, and hold no state that must outlive a reload. - -## Side by side - -| | Cordis and dsh | Chord v2 | -| --- | --- | --- | -| unit | a plugin instance (fiber) in one context tree | an object per (type, input, principal), placed on a host (chapters II, III) | -| side effects | recorded on the fiber with `ctx.effect` and friends; on unload the disposers run concurrently, each as good as its author made it | declared through `self` (documents, natives, routines, jobs, contributions); Chord activates and undoes them (chapters V, IX to XI) | -| dependencies | service names as strings in one flat namespace, via `inject`; a provider reload restarts every dependent | typed contracts via `get(Type, input)`; a reference parks and recovers through restarts (chapter VI); optional dependencies are registries (chapter XIII) | -| ordering | listeners run in registration order, which changes when a plugin reloads | registries compose with a declared `compose` function; nothing depends on load order (chapter XIII) | -| where code runs | one process; remote UIs over RPC to the Host | a tree of hosts (server, workers, tabs); objects reachable across links, relayed by the server (chapters I, XVII) | -| who is calling | not a framework concept | principals stamped by links, variants per caller, revocation (chapters III, VII) | -| state | the plugin's own business; dsh's session log is a service | documents written in transactions, persisted, versioned and migrated (chapter V) | -| reload | unload the old instance, load the new one: in-memory state and running work of that plugin end | pure objects swap their code in place with their documents and running work kept; only objects with natives restart (chapter XV) | -| a running agent turn during reload | entries of changed plugins stop; unloading the agent provider stops its live agents | the harness is base code; Durable pins tools per task phase, so a running call finishes on old code and the next phase uses the new (chapter XIV) | -| crash recovery of a turn | the turn is closed as `interrupted` | pi-durable continues the turn's tasks from their checkpoints; replay-safe tools rerun (chapter XIV) | -| contracts and versions | TypeScript types only, pre-stable APIs, every consumer updated together; config validated with a schema | contract versions with fingerprints and provider-side mappings for old consumers (chapter XVI) | -| status | in production in Koishi since 2019; dsh released | a design with the contract layer built | - -## What the comparison says - -**The core idea is the same.** "Factories declare, Chord activates" (chapter II) and Cordis' reversible effects are -two forms of one rule: a plugin may only change the world through its runtime, so the runtime knows what to undo. -Both reject the module-level timer and the stored callback for the same reason. - -**The difference is what survives a reload.** Cordis reloads at plugin granularity: the old instance unwinds and the -new one starts from nothing, together with every plugin that depends on it. That is simple, but a plugin's in-memory -state and its dependents' state and work in flight end with it, and correctness rests on every disposer and on -registration order. Chord v2 keeps state out of closures on purpose (documents, references, jobs as -records) so that it can swap the code of a pure object while its data and running work stay, and restarts only what -holds OS handles. That is more machinery: transactions, a store, versions on every document and contract. - -**The other difference is reach.** Cordis composes plugins inside one process, and dsh adds RPC to its own clients. -Chord v2 makes the process boundary part of the model: objects live on the host their placement names, callers are -principals, and a reload rolls out across the server, the workers and every tab. An app like p2, with a process per -session and guests in other browsers, needs that; a single-process agent does not. - -:::review | Review: what a reload does to a running tool call in dsh -The sources say that Loader entries running affected plugins stop and that unloading the agent provider stops its live -agents (`packages/boot/hmr/README.md`, `docs/subsystems/core.md`). They do not say what happens to a tool call in -flight when only the tool's plugin reloads; the comparison above does not claim either way. -::: diff --git a/packages/chord/docs/next/explainer/src/21-epilogue.md b/packages/chord/docs/next/explainer/src/21-epilogue.md deleted file mode 100644 index 987fb3c44..000000000 --- a/packages/chord/docs/next/explainer/src/21-epilogue.md +++ /dev/null @@ -1,104 +0,0 @@ - -# What's Next {.epilogue lede="Where Chord v2 stands, what is still undecided, and where the written sources disagree with each other."} - -The guide built Studio from one object to a p2-sized app. Every piece used the same few rules: one owner per thing, -factories that only declare, state that changes only in transactions, long-lived behaviour in declared forms, and -mappings and migrations owned by the provider. Those rules are what let Mario press Reload on his phone in the -preface while the agent keeps running. This last chapter says how much of that exists and what is still open. - -## Status - -:::stats -- ~10k | lines in the frozen first kernel (steps 1 to 6, 443 tests), kept as a reference only -- 1 of 11 | layers of the rebuild started: L1 contracts, built and reviewed -- 6–7k | target size of Chord v2 without tests -::: - -- **The design is decided** except the points marked below. `packages/chord/docs/next/design.md` is the target; - `packages/chord/docs/next/decisions.md` is the log and wins where they differ. -- **The rebuild** goes layer by layer, each with a one-page spec the owner approves, a minimal build with counting and - garbage-collection tests, the cost-guard metrics, and a cleanup loop: L1 data and contracts; L2 readable and - derive; L3 documents and store; L4 objects; L5 references and `ctx`; L6 protocol and links; L7 relay, routes, - revocation; L8 routines and jobs; L9 registries; L10 reload and rollout; L11 telemetry in Chord. -- **Built:** the contract layer (`src/next/contracts/`, commit `7d2bb8ea8`, chapter XVI). In pi: Delta, - `pi-telemetry`, `chord/context`, `chord/telemetry-sqlite`, pi-ai and Durable telemetry, Durable's required - `checkpointWhen`. -- **Next, in order** (`apps/p2/docs/p3-handoff.md`): decide Q12 and Q15, which the L1 spec needs; write the spec for - the rest of L1 (values, inputs and identity, argument checks, principals, errors; draft in - `packages/chord/docs/next/layer-1.md`); then Q11, Q13, Q14 and the cuts C3 to C17 before the layers they affect. -- **In pi:** `mapShared` and bytes as atomic leaves in Delta's `diffRevisions`. -- **Then** the playground web shell (React, esbuild) with the reload control and a preview at `p3.mariozechner.at`, - and the first real proof: the harness, one tool with an approval pause, a code mode reload. Then the HTTP and - channel components, then p3, domain by domain. - -## Deferred decisions - -Kept with stable numbers in `design.md` §26, each waiting for a concrete case: - -| # | Topic | When | Chapter | -| --- | --- | --- | --- | -| 9 | jobs as objects: a record inside the owner, or an object addressed by its id | when login is ported | XI, XIX | -| 11 | where a flow lives: a keyed document of its owner, or an object `LoginFlow { id }` | when login is ported | XII | -| 14 | project plugins: per-directory plugins with trust prompts and snapshots | own pass over `plugin-loading.md` | XV | -| 16 | a Durable Object profile | when a DO target becomes concrete | | -| 17 | session files: one `SessionFiles { session }` owner on the server (proposed) | p3 design | XX | -| 18 | session delete: retire by tag plus a session directory, or owner hooks | p3 design | XIX | -| 20 | connection pools: one owner of SSH connections per environment, or per-use connections | p3 design | XX | -| 21 | config files with a single writer: one owner on the server, others follow | p3 design | XVIII | -| 22 | file formats (`envs.json`, downloads, uploads) as versioned interfaces | p3 design | XX | -| 23 | the terminal host process: inside the revision or deliberately outside | p3 design | XX | - -## Open from review round 3 - -Decisions (`apps/p2/docs/p3-reviews/v2-review3-summary.md`; Q1 to Q10 are decided): - -| # | Problem | Proposal | -| --- | --- | --- | -| Q11 | a singleton that does not exist yet: `undefined` or `initial()`? | singletons read `initial()`; `undefined` only for absent keyed documents | -| Q12 | stripping arguments (R7) against "nothing is copied" | copy on change: a new object only when the argument had extra fields; never mutate the caller's value (proposed in the L1 draft) | -| Q13 | `ctx.incarnation` on relayed calls | relays stamp the origin host's incarnation, as they stamp the principal | -| Q14 | a writable holding functions exposed as a state member | only members with a JSON schema in the contract are exposable, checked at serve | -| Q15 | arguments of `local` methods: validated or not | apply R7 to `local` too, or keep `local` as the one exception | - -Cuts the owner may choose (each removes a primitive this guide teaches): - -| # | Cut | What it costs | -| --- | --- | --- | -| C3 | shrink jobs to a runner over documents (no versions, `accepts`, `blocked`, `persist`), or move jobs out of the kernel | job versioning, or `self.job` for core-only users | -| C4 | routines: `on` and `loop` only | `every`'s due time across reloads | -| C5 | merge `lifetime: "host"` objects into the facet | a plugin's always-on natives share one failure domain | -| C6 | remove `broken()` for opens and `retryOn` | nothing; reload and Retry still work | -| C7 | remove provider-side `linger` | ten lines of app code | -| C8 | remove `state: false` (method-only references) | a method-only consumer pays one snapshot | -| C9 | remove `previous` from derives | derives that used it move to `mapShared` | -| C10 | one `cell(initial)` for `constant` and `writable` | nothing | -| C11 | remove `ctx.trace` (use `traceIn(ctx, …)`) | nothing | -| C12 | remove call credits (the byte window covers calls) | nothing | -| C13 | two timeouts (`closeTimeout`, `readyTimeout`) instead of eight | per-thing overrides | -| C14 | remove the `handle.change` and `handle.retire` shorthands | nothing | -| C15 | remove `inVariant` and `expectVariant` (keep `RefIn`) | nothing | -| C16 | remove facet-owned mutable documents and their transaction exception | atomic writes across an object and its facet move to one owner | -| C17 | no harness-specific start-barrier exception | factories before the barrier cannot depend on the harness | - -## Where the sources disagree - -This guide follows the decision log where it is newer than the design, and the design where it is newer than -everything else. The places that need a fix in the sources: - -:::review | Review: drift in the written sources -- `design.md` §26 lists only the deferred decisions; it omits the open review-3 items Q11 to Q15 and the cuts C3 to - C17. -- `design.md` §3 has a garbled sentence, "Values are method arguments are stripped". -- `apps/p2/docs/p3-handoff.md` §3 is stale: it says all R items are decided and, two lines later, that R2 to R5 and - R7 to R26 are open, and "resume R6 next". -- `kernel.md` and `step-*.md` describe the frozen code: host values, a development freeze, consumer-side linger, - JSONL storage, descriptions on the wire, and a link closed on an unknown id. None of these is the target. -- `chord-next-walkthrough.md` lags the log: validation by trust (now R7, every caller), dev mode and freeze (removed - by R9), descriptions once per link (gone with R6), an IPC frame-count window (now one byte window, Q4), job outcome - `{ ok }` (now `{ kind }`), harness activation as the start barrier (R17), registries "newer refused at load" (Q8), - and hooks using `host.root` (now `chord(fn, context)`, Q6). -- `apps/p2/docs/questions.md` uses `self.resource` and `replace` (both rejected) and `host.root` in hooks. -- `apps/p2/docs/p3-reviews/v2-cost-model.md`: the measurements are current, its "Decisions this needs" section is - not (chapter XVII). -- L1's `Implementation` shape differs from the factory return value used in the design (chapter XVI). -::: diff --git a/packages/chord/docs/next/explainer/src/22-appendix.md b/packages/chord/docs/next/explainer/src/22-appendix.md deleted file mode 100644 index 42eb965f4..000000000 --- a/packages/chord/docs/next/explainer/src/22-appendix.md +++ /dev/null @@ -1,190 +0,0 @@ - -# API Reference {.appendix #api lede="Every name the guide uses, grouped by where you meet it. Working names; only the contract layer is built."} - -## Hosts - -| Call | Meaning | Chapter | -| --- | --- | --- | -| `createHost({ address?, principal?, storage?, principalKey?, revokedBy?, telemetry?, onError?, timeouts })` | a host; with an address it is addressable and its own code acts as `principal`; without, a client | I | -| `host.load([{ name, facet }])` | load facets by plugin name; base code passes its core facet the same way | I | -| `host.connect(link, { principal })` | the link to the parent (at most one) | I | -| `host.accept(link, { principal })` | a link from a client; every request on it acts as `principal` | I | -| `host.route(Kind, { authorize, resolve })` | how to reach hosts of a kind; `resolve` returns `{ link, principal }` | I | -| `host.revoke(r)` | end matching objects and links here and below | VII | -| `host.root(fn, { principal?, signal?, telemetry? })` | a call scope with a `ctx` for code outside objects | VI | -| `host.get(Type, input, { signal, state?, parkTimeout? })`, `host.hold(…)` | a reference from outside objects; `release()` ends it | VI | -| `host.serve(Type, factory, { lifetime?, linger? })` | serve an object type (in a facet) | II | -| `host.registry(Registry)` | the composed value of a registry, a readable | XIII | -| `host.references(Type)` | who holds references to objects of a type on this host (presence) | VIII | -| `host.activity` | `{ references, calls, jobs, lingering }`, for keep-alive | I | -| `host.close()` | end the host: objects, natives, links | I | - -## Inside a factory: `self` - -| Member | Meaning | Chapter | -| --- | --- | --- | -| `self.state(Definition)` | a document handle | V | -| `self.tx(async (tx) => …, ctx)` | a transaction on the object's line | V | -| `self.derive(name, [sources], fn)` | a derived readable | VIII | -| `self.get(Type, input)` | a reference held for the object's life | VI | -| `self.open(open, close, { closeTimeout? })` | a native; `.value` once open | IX | -| `self.routines.on(readable, fn)`, `.every(interval, fn)`, `.loop(fn)`, `.custom(…)` | standing behaviour | X | -| `self.job(Job, { run, finish? })` | owned work that outlives its call | XI | -| `self.contribute(Registry, value)` | a contribution (facets and `lifetime: "host"` objects only) | XIII | -| `self.writable(initial)` | a host-local readable of any in-process value, with `set` | XIII | -| `self.retryOn(readable)` | a trigger that retries a broken activation | VI | - -**Document handles:** `value`, `get(key)` (keyed), `keys`, `change(fn, ctx)` / `change(key, fn, ctx)` and `retire(ctx)` / `retire(key, ctx)` (shorthands for -one-document transactions). **Transactions:** `tx.draft(handle, key?, { tags? })`, `tx.retire(handle, key?)`, -`tx.start(job, args, { key? })`. **Jobs:** `job.start(args, ctx, { key? })`, `job.abort(id)`, `job.running`, -`run.outcome(ctx)`. - -## References and `ctx` - -| Member | Meaning | -| --- | --- | -| `ref.` | a state member (readable: `.value`, subscribe) or a method | -| `ref.state`, `ref.onState(fn)` | `loading`, `live`, `failed` (cause, `retryAt`), `released` | -| `ref.retry()` | retry a failed reference now | -| `ref.release()` | end a reference taken outside objects | -| `ref.members` | the variant's members | -| `ctx.signal`, `ctx.principal`, `ctx.incarnation`, `ctx.get` | cancellation, caller, calling host's incarnation, call-scoped references | -| `ctx.telemetry`, `ctx.trace(name, attributes, fn)` | the current span; a child span with a child `ctx` | -| `ctx.jobId`, `ctx.progress(value)` | in a job's `run` | - -## Definitions - -| Name | What it declares | Chapter | -| --- | --- | --- | -| `hostKind(name, input?)` | a kind of host and its address | I | -| `remote(name, { version, input, placement, variants })` | an object type reachable across links | II | -| `local(name, { version, input, members })` | an object type on its own host | IV | -| `method(args?, result?)` | a method in a base declaration | II | -| `evolve(prev, (e) => …)`, `checkAccepts(contract, accepts)` | the next contract version; the provider's mappings (**built**) | XVI | -| `Implementation` | the type of a provider for the current version (**built**) | XVI | -| `str num int bool nul lit oneOf union obj arr map opt bytes unknown`, `Static` | schema helpers (**built**) | II | -| `defineState(name, { schema, version, initial, checkpointWhen, persist?, keyed?, ordered?, migrate? })` | a document | V | -| `defineTag(name)` | a tag; `store.retireByTag(tag, value)` | V | -| `defineJob(name, { version, args, progress, result, concurrency?, persist?, accepts? })` | a job type | XI | -| `defineRegistry(name, { version, compose, accepts? })` | an extension point | XIII | -| `bindFacet

()` | base code's `defineFacet`, with factories' principals typed as the app's `P` (working name) | III | -| `defineFacet((host, self) => …)` | a plugin's setup for one kind of host; imported from base code (`@studio/core`) | II, III | -| `constant(value)` | a readable that never changes (start data) | IX | -| `broken(error)` | a failure that is not retried by the backoff | VI | -| `inVariant(ref, …)`, `expectVariant(ref, …)`, `RefIn<…>` | narrowing a reference by variant | III | -| `mapShared(fn)` | an array map that keeps unchanged items identical (planned in Delta) | VIII | -| `OutputBuffer`, `Progress` | bounded output and throttled progress (`@earendil-works/chord/output`) | XI | -| `memoryLinkPair()`, `ipcLink(child)`, `webSocketLink(ws)` | link adapters | I | -| `DocumentStorage`, `registerDocumentStorageConformance` | the storage port and its shared test suite | V | - -# Timeouts and Defaults {.appendix #defaults} - -| Timeout or default | Bounds | Default | Override | -| --- | --- | --- | --- | -| `closeTimeout` | a native's `close`, then reported as a possible leak | 30 s | per native | -| call drain | an ending object's own call handlers | = `closeTimeout` | per type | -| `readyTimeout` | activation; an awaited `get` | 60 s | per `get` | -| park deadline | parked calls (link drop, loading, rollout) | 60 s | per `get` (`parkTimeout`); per type | -| routine handover | a new routine waiting for the old handler on reload | 10 s | per routine | -| `finish` retry | backoff of a throwing job `finish` | 1 s to 60 s | per job | -| rollout install | one host installing a revision | 60 s | none | -| linger | an object kept alive after its last reference | per type | per type | -| backoff | retries of transient failures, per address | 1 s doubling to 60 s | none | -| eviction | keyed persisted documents idle in memory | 10 minutes | | -| client WebSocket limits | per part | 16 MiB compressed, 64 MiB decoded | per link kind | -| compression | JSON parts compressed at their origin | above about 16 KiB | | -| worker keep-alive (p2, app code) | idle time before the supervisor stops a worker | 5 minutes | app | - -# Versions {.appendix #versions} - -| Surface | Version | On change | Chapter | -| --- | --- | --- | --- | -| object type | `version` plus fingerprint | `evolve` and `checkAccepts`; otherwise `contract_mismatch` | XVI | -| document | `version` plus schema fingerprint | pure `migrate` on access; no bump fails `broken` | V | -| job | `version` | `accepts` on outcomes and `running`; `blocked` without | XI | -| registry | `version` | `accepts` on contributions; unmatched ones left out and reported | XIII | -| native | none | restarts on reload | IX | -| Durable tasks and documents | Durable's `version` | Durable's `migrate` | XIV | -| link protocol | protocol version | refused; hosts update together | XVII | -| UI components, Chord and app APIs | none yet | exposed as they are | | - -# Errors and States {.appendix #errors} - -`ChordError` has a stable `code` and a finer `reason`; on the wire it is `{ code, reason?, message, details? }` -(specified in `layer-1.md`, not built). - -| Code | Meaning | -| --- | --- | -| `invalid` | input, arguments or placement rejected (`invalid/input`, `invalid/args`, `invalid/placement`) | -| `contract_mismatch` | the consumer's version and fingerprint are neither current nor accepted | -| `forbidden` | the caller may not do this | -| `gone` | the object or hold ended | -| `not_ready` | not activated in time | -| `released` | the reference was released | -| `too_large` | a part exceeded its size limit | -| `unavailable` | could not be reached before the deadline (`unavailable/locked` and others) | -| `aborted` | cancelled by a signal | - -| Type | Values | -| --- | --- | -| `FailureCause` | `{ kind: "transient"; error }` (retried with backoff) or `{ kind: "broken"; error }` (retried only by a trigger) | -| reference state | `loading`, `live`, `failed` (cause, `retryAt`), `released` | -| job outcome | `{ kind: "done"; result }` or `{ kind: "failed"; error: "threw" \| "aborted" \| "lost" }` | -| job phase | `running`, `finishing`, `blocked` | -| rollout status of a provider | `updating` (calls park), rejected (fail at once), same revision (`contract_mismatch`) | -| host install report | `installed`, `installed_failed`, `rejected` | - -# Rejected Designs {.appendix #rejected} - -From `design.md` §31, grouped. Each was considered and is not coming back without a new reason. - -| Area | Rejected | -| --- | --- | -| reload | drain-based handover; object restart for state changes; a factory trial; a migrate dry run; a consumer contract check during staging; reload votes; per-plugin rollback within a host; keeping natives across reloads by name; `restart_required`; host values and a development freeze | -| state | version fences without transactions; volatile documents; per-group lines; optimistic transactions; `tx.get`; a JSONL document storage; validating state and documents on commits or reads; copying values on same-host calls; op-rewriting projections and a view algebra; Durable's Session as the object store | -| objects and references | resources replaced in place (`replace`, `deps`, stale handles); revoking proxies on `.value`; consumer-side linger; detached references counting toward presence; an `optional` flag on references; `opaque` members; callbacks as method arguments; generic call or request ids; a stream type in Chord | -| registries | registries created at runtime; entries resolved through mutable indirection; generic registry keys with a `conflict` flag; numeric middleware priorities; registry composition as a staging check | -| principals and placement | principal field patterns and re-stamping for revocation; `storeKey`; placement as data naming an input field; host-name strings in app code; "magic" input-field matching for delete | -| wire | an outbox for transient delivery errors; cross-relay reset coordination; count limits that refuse calls or holds; contracts taken from the running app | -| process and tooling | a topology layer above the host API; loader variants A and B; Chord ending the process on a storage failure; `defaultProjectTrust: "always"`; ceremony (`native/` directories, `defineNative`, lints); weakening type checking | - -# Glossary {.appendix #glossary} - -**Adapted binding:** a consumer bound at an older accepted contract version, served through the provider's mappings. -**Base code:** the app's own code, imported statically; a change restarts the hosts. **Client:** a tab or TUI host, -not addressable. **Contract:** an object type's declaration (name, input, placement, members, version). -**Contribution:** a value a plugin adds to a registry. **`ctx`:** what every call receives. **Derive:** a value -computed from readables. **Document:** a named JSON value an object owns. **Facet:** a plugin's code for one kind of -host. **Factory:** the synchronous function that declares an object. **Fingerprint:** a hash of a contract's or -schema's normal form. **Flow:** input into running work, kept as state by its owner. **Host:** one participant of the -app, with an identity and an incarnation. **Incarnation:** the id a host gets on every start. **Input:** an object's -key among objects of its type; with the type and the caller's principal key it identifies the object. **Job:** owned work that outlives its call. **Link:** an ordered connection between -two hosts. **Local type:** an object type reachable only on its own host, without principals. **Native:** an OS-level -value opened with `self.open`. **Park:** a call waiting for a reference to become live again. **Placement:** the -function from input to host address. **Plugin:** a directory with a `package.json`, reloadable. **Principal:** who -calls, stamped by the link. **Principal key:** the string that identifies a principal in object identities. **Pure -object:** an object without natives; swapped in place on reload. **Readable:** a value plus its ordered changes. -**Reference:** what `get` returns. **Registry:** an owner's extension point. **Relay:** a host forwarding messages -between two links, rewriting only headers. **Reload:** swapping a plugin's code in running hosts. **Restart:** ending a -process. **Revision:** the complete code set the hosts should run. **Revocation:** ending live access for principals -matched by `revokedBy`. **Routine:** standing behaviour of an object. **Server:** the addressable root host. **Shared -store:** a local object that holds data once behind per-principal views. **Tag:** a typed label on documents, used to -retire them together. **Transaction:** an atomic set of document writes on the object's line. **Variant:** the set of -members a principal gets. **View:** a per-principal remote object in front of a shared store. **Worker:** a host the -server starts for one session. - -# Sources {.appendix #sources} - -| Source | Used for | Weight | -| --- | --- | --- | -| `packages/chord/docs/next/decisions.md` | every decision, review rounds 2 and 3 | wins where newer | -| `packages/chord/docs/next/design.md` | the target design, all chapters | authoritative otherwise | -| `packages/chord/src/next/contracts/` and its `README.md` | chapter XVI | built code | -| `packages/chord/docs/next/layer-1.md` | errors, inputs, argument checks | draft spec | -| `apps/p2/docs/p3-reviews/v2-review3-summary.md` | open Q11 to Q15, cuts C3 to C17 | current | -| `apps/p2/docs/p3-reviews/v2-cost-model.md` | measurements, per-hop costs | current except its last section | -| `apps/p2/docs/p3-functions/` | chapters XVIII to XX | current | -| `apps/p2/docs/p3-handoff.md` | status and order of work | §3 partly stale | -| `packages/durable/README.md` | the harness, submit, tasks, subagents | current | -| `chord-next-walkthrough.md`, `kernel.md`, `step-*.md`, `apps/p2/docs/questions.md` | background only | lag the decisions (see What's Next) | -| DeepSeek Harness at `687f282` (its Cordis tutorial, architecture, persistence, core and HMR docs), the Cordis paper (arXiv 2608.25512) | "Other Ways to Do It" | external, built systems | diff --git a/packages/chord/docs/next/explainer/walkthrough.md b/packages/chord/docs/next/explainer/walkthrough.md deleted file mode 100644 index e4bfa5a9e..000000000 --- a/packages/chord/docs/next/explainer/walkthrough.md +++ /dev/null @@ -1,284 +0,0 @@ ---- -title: Chord v2 -kind: walkthrough -kicker: Walkthrough · Chord v2 -subtitle: Build an app like p2 on Chord v2 and pi-durable, one concept at a time, with a tutor that draws the figures and asks the questions -model: anthropic/claude-sonnet-5-5 -sources: - - packages/chord/docs/next/explainer/book.md - - packages/chord/docs/next/design.md - - packages/chord/docs/next/decisions.md - - packages/chord/docs/next/layer-1.md - - packages/chord/src/next/contracts/README.md - - packages/chord/src/next/contracts/*.ts - - packages/durable/README.md -truth: treat Chord v2 as fully built as designed; ignore what the sources say about build status. redesign-decisions.md wins over design.md; design.md wins over the guide (book.md) ---- - -:::tutor -The learner knows pi-durable (harness, conversations, tasks, tools) and nothing about Chord. - -Teach Chord v2 as a finished, working runtime: never say a part is "not built", "planned" or "the target design", even -though the sources say so in places. Open decisions may be mentioned as "still being decided" when a concept depends on -one. - -The running example is Studio, introduced on the page before the first concept (see the introduction). Use it in every -concept and grow it step by step, as the guide book.md does chapter by chapter: read the matching guide chapter before -each concept, and check details in design.md and the decision log. Name the guide chapter in "Read more" lines. - -Keep code short and complete enough to read on its own. Prefer one figure per concept: a sequence for message orders, -a flow for structure, a stepper for lifecycles. -::: - -:::intro -#### What this is about - -Picture **p2** (Pi Mobile): a web app for working with pi agents from any device. It runs on your laptop as one -**server** process. Every chat session gets its own **worker** process, and in each worker a pi-durable agent works -through its turns. You use it from **browser tabs** on your desktop or phone, and you can invite a colleague to watch. -Most features (a notes panel, file browsing, terminals, provider logins) are **plugins**. - -Now you change one line in a plugin and press Reload, while an agent is twenty minutes into a task. The panel should -show the new code on every screen. The agent's turn, the running tool, the open connections and every bit of saved -data should simply continue. No restart, no lost work. - -**Chord v2** is the runtime that makes this normal. An app on Chord is a set of **objects** spread over the server, the -workers and the tabs. Each object owns its data and its OS handles, declares what it needs, and Chord creates, -connects, reloads and ends it. Because Chord knows every piece of state an object has, it can swap the object's code -while its data stays. - -#### The example: Studio - -This walkthrough builds **Studio**, a small p2, on Chord v2 and pi-durable. It has the same three kinds of processes: - -:::flow wide | Studio: one server, a worker per session with its agent, browser tabs as clients. The notes plugin is the first feature we build. -row "Clients" muted -box owner "owner's tab" "phone or desktop" -box guest "guest's tab" "read-only" -row "Server" accent -box server "server" "session list, access" accent -row "Workers" blue -box worker "worker S1" "agent + notes" blue -box worker2 "worker S2" "agent + notes" blue -arrow owner -> server "WebSocket" -arrow guest -> server "WebSocket" -arrow server -> worker "IPC" -arrow server -> worker2 "IPC" -::: - -The first feature is a **notes plugin**: a panel where the owner adds notes to a session, a guest reads them, and the -agent can add notes through a tool. Each concept adds one piece to Studio, until it has everything a p2 needs: who may -do what, saved data, live updates, background jobs, logins, extension points, the agent, and reload. - -#### How it progresses - -1. **The pieces:** processes, objects, callers, shared data, saved data. -2. **Living objects:** references that survive restarts, revocation, computed values, OS handles, standing behaviour, - background jobs, input from people. -3. **The app:** extension points between plugins, and the pi-durable agent inside Chord. -4. **Change:** reloading code while everything runs, evolving contracts, and what goes over the wire. -::: - -# What we're building {#goal part="The pieces"} -Guide preface ("Before we begin", "Ten things to know"). Must cover, briefly and concretely: the reload scenario (phone, -agent mid-turn, guest watching, one-line fix, Reload) and why restarting processes or hot-swapping modules fails; the -one rule behind Chord (every piece of state, OS handle and running work has one owner, an object, and objects only -declare what they need); Studio's pieces from the introduction. Do not go into APIs yet. - -:::check -1. Why is restarting the worker processes not a good way to load new plugin code? -+++ answers -1. It loses the agent's turn in flight, open terminals and connections (pi-durable would resume the turn, but the running tool and the connections are gone). -::: - -# The shape of an app {#shape part="The pieces"} -Guide chapter I. Must cover: hosts (server, workers, clients), links, host kinds and placement, createHost/load/connect/accept/route, -clients reach workers only through the server, incarnations, keep-alive is app code (host.activity). - -:::check -1. A tab wants an object that lives on worker S1, which is not running. What happens, in order? -2. Why does Chord have no listen() call? -+++ answers -1. The tab's call goes to the server; the address is not the server's, so it uses route(Session): authorize, then resolve starts the worker and returns a link plus principal; the server relays the hold, the worker builds the object and answers with a snapshot. -2. Entry points differ too much (loopback WebSocket, Radius channels, a Durable Object edge, memory links); each authenticates and calls accept. -::: - -# A first object {#first-object} -Guide chapter II. Must cover: contract (name, input, placement, variants, version): the input is the object's key -among objects of its type, `placement` computes the host from it; schema helpers, facet and factory (a facet imports -Studio's own defineFacet from @studio/core), serve and get, methods return data, placement checked on both sides. - -:::check -1. Two tabs of the owner ask for Notes of S1. How often does the factory run? -2. A caller with an outdated placement rule asks worker S2 for Notes of S1. What happens? -+++ answers -1. Once per principal key; the owner's two tabs on the same access path share the object, so once. -2. The provider runs the placement function itself and refuses with invalid/placement. -::: - -# Who is calling {#principals} -Guide chapter III. Must cover: the app's principal type, links stamp principals, principal key in the object identity, -variants per principal, narrowing with inVariant/RefIn, and how a facet knows P: base code exports defineFacet bound to -P (bindFacet

()), every facet imports it, and host.load accepts only facets typed for its P. - -:::check -1. Can a client send a principal field to claim it is the owner? -2. Why do guests not get an add method that throws "forbidden"? -3. The notes plugin's worker facet narrows principal.kind === "guest". Where does the type checker get Studio's principal union from? -+++ answers -1. No: a client's principal field is always overwritten by the link's stamp; only a trusted parent link may carry one. -2. Variants: the reader variant simply has no add member, and Chord refuses calls to members outside the caller's variant. -3. From Studio's base code: @studio/core exports defineFacet = bindFacet

(), the facet imports that one, and the worker's host.load accepts only facets typed for the same P. -::: - -# One store, many views {#shared-store} -Guide chapter IV. Must cover: local types, the shared store pattern, attribution as data, argument validation for every caller (R7, Q15 open). - -:::check -1. Where does "who wrote this note" come from if the store is local and has no principal? -+++ answers -1. The remote view knows the caller and passes the author into the store's method as data. -::: - -# Writing it down {#documents} -Guide chapter V. Must cover: defineState, self.tx, drafts and ops, persistence only for stable identities, versions with migrate, tags, storage. - -:::check -1. Why may a remote per-principal object not persist documents? -2. A schema changes but the version is not bumped. What happens? -+++ answers -1. Its identity includes the principal key, which changes (devices, grant revisions); persisted data needs a stable identity, so it lives in a local store or a host-lifetime object. -2. The stored fingerprint differs from the definition's at access, and the object fails broken ("schema changed without a version bump"). -::: - -# Holding on {#references part="Living objects"} -Guide chapter VI. Must cover: reference lifecycle, ref.state, parking, transient vs broken failures, backoff, ctx. - -:::check -1. The link to a worker drops while a tab calls a method. What does the call do? -+++ answers -1. A call already in flight fails like on a disconnect; calls made while the link is down park until the reference is live again (up to the park deadline, 60 s), then fail unavailable. -::: - -# Taking access away {#revocation} -Guide chapter VII. Must cover: revokedBy, host.revoke, propagation to children, no re-stamping, lasting denial in app data. - -:::check -1. After revoke({ grant: "g1" }), Ann reloads her tab. Why does she not get access back? -+++ answers -1. revoke cuts only live access; the app's guest record is blocked and authenticate refuses it on the new connection. -::: - -# Computed values {#derives} -Guide chapter VIII. Must cover: derives, the footgun (new objects for unchanged data), mapShared, container width, presence with host.references. - -:::check -1. Why is items.map(stripAuthor) in a derive expensive on every commit? -+++ answers -1. It builds new objects for every unchanged note, so the diff walks and sends O(value) instead of O(change); mapShared keeps unchanged items identical. -::: - -# Things the OS gives you {#natives} -Guide chapter IX. Must cover: natives with self.open, pure vs native objects, start data with constant, lifetime host, call-scoped natives. - -:::check -1. What happens to a native object on reload, and why? -+++ answers -1. It restarts: its natives close and reopen with the new code, because old and new code sharing an OS handle would interleave unchecked. -::: - -# Standing behaviour {#routines} -Guide chapter X. Must cover: routines (on, every, loop), serialized handlers, reconciling instead of notifying, pub/sub reports with ctx.incarnation. - -:::check -1. Why does archiving a session not call notes.archive(session) directly? -+++ answers -1. A notification can be lost (worker not running, link drops, crash); a routine on the catalog's sessions reconciles from the current value, also after a crash. -::: - -# Work that outlives the call {#jobs} -Guide chapter XI. Must cover: defineJob, the intent / effect / outcome sandwich, dedupe of unsettled runs (Q9), finish in current code, abort as a request. - -:::check -1. A shell job's run crashes after writing the result entry. How is a duplicate entry avoided? -+++ answers -1. The external write carries a key derived from ctx.jobId (requestId shell:jobId), so Durable dedupes it. -::: - -# Asking a person {#flows} -Guide chapter XII. Must cover: a flow is state, not a call; steps helper; the Questions service; decisions #9 and #11 open. - -:::check -1. Why not a long-held step() call that returns the next login step? -+++ answers -1. A link drop fails it and the consumer cannot tell if it missed a step; state re-holds by itself and shows the current step. -::: - -# Extension points {#registries part="The app"} -Guide chapter XIII. Must cover: registries, compose, who may contribute (R21), contributions that change, optional dependencies, HTTP component, channels, chunked transfers. - -:::check -1. Radius needs forwards from the ports plugin, which may be missing. How is that expressed? -+++ answers -1. Radius owns a registry (radius.forwards) and ports contributes to it; without ports the table is empty. -::: - -# The agent {#agent} -Guide chapter XIV. Must cover: the Harness as a core host-lifetime object, HarnessModels / HarnessEnvironments / HarnessExtensions registries, chord(fn, context) for tools, approval via Questions. - -:::check -1. How does a Durable tool get a Chord ctx? -+++ answers -1. Through chord(fn, context): the host is in Durable's context under a context key, and host.root gives fn a ctx. -::: - -# Reload and rollout {#reload part="Change"} -Guide chapter XV. Must cover: load, staging, commit, per-object swap or restart, rollback all-or-nothing per host, revisions, rollout order, updating vs contract_mismatch. - -:::check -1. A staging error happens in one of three plugins being reloaded on a host. What runs afterwards? -+++ answers -1. The whole host stays on its previous code; rollback is all-or-nothing per host. -::: - -# Changing contracts {#contracts} -Guide chapter XVI; the code is in src/next/contracts. Must cover: evolve, checkAccepts and mapping directions, binding by version and fingerprint, new history bases, the fingerprint normal form. - -:::check -1. Which way does a state mapping go, and which way an args mapping? -2. Why can an input change not be adapted? -+++ answers -1. State: current to old (the old consumer reads). Args: old to current (the old caller sends). -2. The input is the object's address; changing it changes what the object is, so it needs a new history base and old consumers get contract_mismatch. -::: - -# On the wire {#wire} -Guide chapter XVII. Must cover: messages and parts, relays rewrite only headers, one state message per commit per link, flow control (R19, Q3), size limits per part, the cost model, telemetry. - -:::check -1. What does the server do with a state part it relays from a worker to a tab? -+++ answers -1. It reads and rewrites only the header (maps ids, groups parts per destination link) and forwards the part without decoding it. -::: - -# Other ways to do it {#compared part="In context"} -Guide chapter "Other Ways to Do It" (after chapter XX). Must cover: Cordis (Koishi's plugin framework since 2019) and -DeepSeek Harness (dsh, everything is a plugin, built on Cordis): reversible effects on a plugin fiber, inject-based -services, reload by unload-and-load; dsh's durable session log and its "interrupted" crash repair. Then what the -guarantees are in practice (the chapter's section of that name): disposers run concurrently and are only as good as -their authors, listener order follows registration order and changes on reload, services are untyped strings and a -provider reload restarts all dependents, and reloading a core service stops live agents. Then the two differences that -matter: what survives a reload (Cordis: nothing of the plugin's or its dependents' in-memory state; Chord: a pure -object's data and running work) and reach (one process vs. a tree of hosts with principals). Say what Chord costs: -more concepts and machinery. A flow or table comparing a reload in both helps. - -:::check -1. A Cordis plugin keeps a list of notes in a field of its instance. What happens to the list when the plugin hot-reloads, and what would the same plugin do on Chord v2? -2. Two Cordis plugins register waterfall listeners on the same tool-policy event; the first one short-circuits. What changes after the first plugin is hot-reloaded? -3. Why does an app like p2 need Chord's multi-host model, when dsh runs its plugin tree in one Host process? -+++ answers -1. The old fiber unloads and the new one starts empty, so the list is gone unless it was kept in a service or on disk. On Chord the notes are a document of a pure object; a reload swaps the object's code and keeps the document. -2. Its listener is registered again at the end of the list, so the other plugin's listener now runs first and the decision can change, unless the listener was registered with prepend. -3. p2 has a worker process per session and clients (including guests) in other browsers; objects must live on the right host, callers must be stamped principals, and reload has to roll out across server, workers and tabs. A single-process tree does not cover that. -::: - diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md new file mode 100644 index 000000000..5f9576170 --- /dev/null +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -0,0 +1,733 @@ +--- +title: Hosts and Plugins: a tutorial for Chord v2 +cover: Hosts / and Plugins +kicker: A guide · Chord v2 · 2026-11-02 +subtitle: Build a small notes app step by step, from one host to plugins on a server and on workers, edited while it runs +stats: + - 3 | kinds of host in an app: server, worker, client + - 1 | file per host kind in a plugin + - 0 | restarts to apply a plugin edit + - 1 | rule: plugins talk only through object types and registries +note: | + A tutorial for people building apps on Chord v2, written by the redesign session from `design.md`, `slice-7c.md`, + `slice-10.md`, `slice-12.md`, `slice-13.md`, the decision log and the code in `packages/chord/src/next` and + `src/next-node`. API names are the current ones. The early chapters wire hosts by hand to show the parts; the last + chapter shows the app kit (`chord/next/app`), which does that wiring for an app. +colophon: Hosts and Plugins · third edition · 2026-11-02 · Chord v2 slices 1-10, 12 and 13 +--- + +# Before we begin {.preface} + +You want to build a notes app. You type a note in a browser tab, and every other tab sees it at once. Each project +gets its own notes in its own process, so a project that misbehaves cannot take the others down. And you, or an agent +you work with, add features while the app runs: a search, an export button, a word count, without restarting +anything. + +Chord gives you three things for that. **Hosts**: the processes of your app (the server, a worker per project, the +browser tabs), connected to each other. **Objects**: the state and methods your app is made of, living on one host and +used from any other. **Plugins**: directories of code that declare objects for each kind of host, loaded and +reloaded while the app runs. + +This book builds the app in that order. Each chapter adds one concept and shows complete code. + +## Seven things to know + +:::findings +1. **A host is one participant.** The server, each worker, each browser tab and each terminal UI is a host; hosts + connect over links. Tabs and terminal UIs are clients: no address, they connect to the server and ask. +2. **An object has a type, and a type is a contract.** The contract names the object's state and methods; any host can + ask for an object by type and params, and Chord finds where it lives. +3. **A facet declares, an object acts.** A facet is a plugin's code for one host kind: it says which object types this + host serves. The objects hold the state and do the work. +4. **A plugin is a directory** (or a single file). A `package.json` with the plugin's name, and one file per host kind: + `server.ts`, `worker.ts`, `web.tsx`. It imports its host only through the app's versioned API (`@app/api/v1`). +5. **The server builds, every host evaluates.** The plugins component builds each facet into a small script; the + server, workers and tabs load the scripts for their kind. +6. **Plugins meet through types and registries.** One plugin uses another's objects through its contract, or + contributes to a registry another defines. Nothing else crosses between plugins. +7. **An edit is a rollout.** Change a file, build, roll out: the server, then every worker, swap the plugin's objects in + place; a broken edit keeps the old code running and reports why. +::: + +:::flow wide animate | How it all fits: plugins on disk are built once on the server; each host evaluates the facets of its kind; tabs and the TUI are clients that reach workers' objects through the server. +row "On disk" muted +box disk "plugins/notes/" "`server.ts`, `worker.ts`, `web.tsx`, `tui.ts`, `contract.ts`" muted +box core "src/ (base code)" "`@app/api/v1`: Chord's surface, host kinds, principals" muted +row "Server process" accent +box comp "plugins component" "discover, build every facet, watch, roll out" accent +box server "server host" "server facets; routes Project → workers" +row "Clients and workers" +box tab "client: tab or TUI" "web or tui facets, local UI objects; holds `Notes { p1 }`" blue +box w1 "worker p1" "worker facets; serves `Notes { p1 }`" +arrow disk -> comp "read, build" +arrow core -> comp "host packages" +arrow comp -> server "server scripts" +arrow comp -> tab "web or tui scripts" +arrow tab -> server "hold Notes { p1 }" +arrow server -> w1 "start (worker scripts), relay" +note "Each host loads only its own kind's facets. Plugins meet only through object types and registries." +::: + +## Words used throughout + +A **host** is one participant of the app, made with `createHost`. A **host kind** (`Server`, `Session`) describes +addressable hosts; an **address** (`Session({ id: "p1" })`) names one. An **object type** is a contract: `local` (used +on its own host only) or `remote` (used from any host). An **object** is one instance of a type, per params. A +**facet** is a plugin's declarations for one host kind. A **plugin** is a directory with a `package.json`. A +**reference** is what you get when you ask for an object (`host.get`, `self.get`, `ctx.get`). A **document** is data an object +stores, changed in **transactions**; a **table** holds many records with queries, in the host's **store**; a **derive** is a value computed from others; a **native** is an outside +resource an object opens and closes. A **registry** is an +extension point: one plugin defines it, others contribute. A **rollout** applies a build to the hosts in order. + +# One Host {#host lede="The smallest app: one process, one object type, one call."} + +Start without plugins. One host, one object that keeps notes: + +```ts +import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next"; + +const NoteList = obj({ items: arr(str()) }); // one schema, used by the contract and the document + +// the contract: what a Notes object offers +const Notes = local("notes.store", { + version: 1, + params: {}, // one object per host: no params + members: { + list: NoteList, // state anyone can read and watch + add: method(obj({ text: str() })), // a method: arguments checked by Chord + }, +}); + +// the document the object stores (its own version: storage can change without changing the contract) +const NotesDoc = defineState("notes", { schema: NoteList, version: 1, initial: () => ({ items: [] }) }); + +// the facet: which types this host serves, and how to make each object +const facet = defineFacet((host) => { + host.serve(Notes, (_params, self) => { + const doc = self.state(NotesDoc); + return { + list: doc, + add: async ({ text }, ctx) => self.tx(async (tx) => { (await tx.draft(doc)).items.push(text); }, ctx), + }; + }); +}); + +const server = createHost({ onError: console.error }); +await server.load([{ name: "@app/notes", facet }], ctx); +const notes = await server.get(Notes, {}, ctx); // a reference: live, with its state +await notes.add({ text: "buy milk" }, ctx); +console.log(notes.list.value); // { items: ["buy milk"] } +``` + +Four things happened. The **contract** `Notes` says what the object offers; the **document** says what it stores. +Here they have the same shape, so one schema serves both; they are separate because they change separately (an +object may offer a computed value or a shortened view instead of what it stores, and storage can change its shape +without users noticing). +The **facet** says this host serves +`Notes`, with a **factory** that declares the object's parts: a document, and a method writing it in a transaction. +`load` publishes the facet. `get` returns a **reference** once the object is live; its members are the contract's. + +The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens +what it asked to open) and then serves calls. That is why the same factory can run again when the code changes. + +# An Object's Parts {#parts lede="Documents, tables, transactions, computed values, watching, and outside resources: the vocabulary of every factory."} + +A factory builds an object from a handful of parts, each declared with one call on `self`. The notes object grows to +show all of them. + +**Documents** hold the object's data: `self.state(NotesDoc)` gives a handle whose `.value` is the current data. Data +changes only in a **transaction**: `self.tx(fn, ctx)` runs `fn` with `tx`, and `await tx.draft(handle)` gives a +mutable copy of the document's data; whatever you change on it is committed at once when `fn` returns, or not at all +if it throws. + +```ts +add: async ({ text }, ctx) => self.tx(async (tx) => { + const draft = await tx.draft(doc); // the document's data, writable inside this transaction only + draft.items.push(text); // recorded as a change, committed when the function returns +}, ctx), +``` + +Documents live in memory unless they are **persisted**: `defineState(..., { persist: true, checkpointWhen })` keeps +one in the host's **store** (a SQLite file the app opens and gives to `createHost({ store })`), loaded when the +object activates. For many records with queries, an object declares a **table**: + +```ts +const NoteRows = defineTable("notes", { + version: 1, + columns: { id: str(), text: str(), at: int() }, + key: ["id"], + indexes: { by_time: ["at"] }, +}); + +host.serve(NoteStore, (_params, self) => { // a local type with empty params: it may persist + const rows = self.table(NoteRows); // created (or migrated) when the object activates + const recent = query(NoteRows).orderBy((n) => n.at, "desc"); + return { + add: async ({ text }, ctx) => self.tx(async (tx) => tx.put(rows, { id: uuid(), text, at: Date.now() }), ctx), + page: async ({ after }, ctx) => rows.page(recent, 50, ctx, { after }), // 50 at a time, with a cursor + }; +}); +``` + +Persisted data lives only in `local` objects with empty params (one per host and type), so its address never depends +on who asked or with which params; an object with params (a view per project) reads from such a store. Writes go +through the same transactions as documents: a note row and a document can change in one commit. + +A **derive** is a value computed from other values, recomputed whenever they change. It is how an object offers +something it does not store: + +```ts +const count = self.derive([doc], ([notes]) => notes.items.length); // 0, 1, 2, … as notes are added +return { list: doc, count, add }; // `count` can be a member like `list` +``` + +**Watching** is how anyone follows a value as it changes: a document handle, a derive, a reference's member. `watch` +gives the current value, then one call per change: + +```ts +const watch = await notes.count.watch(ctx); +watch.start(async (call) => { + if (call.kind === "value") console.log("notes:", call.value); // "failed" calls say a value cannot be computed now +}); +``` + +A **routine** is the object watching on its own behalf: `self.on(source, fn)` runs `fn` with each new value, for as +long as the object lives (a reconciler: "whenever the notes change, rebuild the export file"). + +A **native** is something outside Chord the object opens and must close: a file watcher, a database connection, a +child process. `self.open(open, close)` opens it while the object activates and closes it when the object ends: + +```ts +const inbox = self.open(async (ctx) => watchDirectory(inboxPath, ctx.abortSignal), (watcher) => watcher.close()); +self.onLive(async (ctx) => { // a routine started once the object is live + for await (const file of inbox.value.files(ctx.abortSignal)) await importNote(file, ctx); +}); +``` + +Natives are why some objects restart when their plugin changes: an open connection cannot be moved to new code, so +Chord closes it and opens it again. Objects without natives are swapped in place. + +# Several Hosts {#hosts lede="A server, a worker per project, browser tabs: how they find each other's objects."} + +Now give each project its own process. Declare the kinds of addressable host, and make notes a `remote` type that +lives on its project's worker: + +```ts +export const Server = hostKind("server"); +export const Project = hostKind("project", { id: str() }); + +export const Notes = remote("notes", { + version: 1, + params: { project: str() }, + hostAddress: ({ project }) => Project({ id: project }), // where an object of these params lives + variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } }, +}); +``` + +Two things are new. A `remote` type can be used from any host, and it has **variants**: different sets of members +for different callers, chosen by the factory (an `editor` variant with `add`, a `reader` variant for guests without +it). And `hostAddress` says where its objects live; it is the whole routing story. + +A tab asks for `Notes { project: "p1" }`. Its host computes the address, +`Project(p1)`, and sends the request to its parent, the server. The server has a **route** for `Project`: it starts (or +finds) the worker for `p1` and passes the request on. The worker serves `Notes`. The tab gets a reference whose state +follows the worker's document, one frame per change. + +:::flow wide animate | A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data. +row "Clients" blue +box tab1 "tab A" "holds Notes { project: p1 }" +box tab2 "tab B" "holds Notes { project: p1 }" +row "Server" accent +box server "server" "route Project → worker" +row "Workers" muted +box w1 "worker p1" "serves Notes { p1 }" +box w2 "worker p2" "serves Notes { p2 }" +arrow tab1 -> server "hold" +arrow tab2 -> server "hold" +arrow server -> w1 "relay" +::: + +Each host is created the same way, with what it is: + +```ts +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); + +// worker.ts (started by the server for one project) +const worker = createHost

({ address: Project({ id }), principal: { kind: "worker", project: id }, onError }); +worker.connect(parentLink, { principal: { kind: "server" }, trusted: true }, ctx); + +// a browser tab: no address, it is a client +const tab = createHost

({ onError }); +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +``` + +`P` is your app's type of **principal**, who is calling: an owner, a guest, the server, a worker. Links stamp it; +a `remote` object is made per principal, so a guest can get a read-only variant of the same notes. + +:::note | The app kit does this wiring +Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components +(`createWorkers`, `chord.http`). An app built with the app kit (`chord/next/app`, the last chapter) does not wire them: +`startServer(...)` on the server and `startWorker(...)` in a worker's entry do. +::: + +# Client Hosts {#clients lede="Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot."} + +A browser tab and a terminal UI are hosts like the server, with one difference: they have **no address**. Nobody can +send a request *to* a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab +runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state. + +```ts +// a browser tab (slice 11's page host will do this wiring for you) +const tab = createHost

({ onError }); // no address, no principal: a client +tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx); +await tab.load([{ name: "@app/web", facet: webCore }, ...webFacets], ctx); + +const notes = tab.hold(Notes, { project: "p1" }, ctx); // a reference at once, live soon +(await notes.list.watch(ctx)).start(async (call) => { + if (call.kind === "value") render(call.value.items); // every change, as it commits on the worker +}); +await notes.add({ text: "from the tab" }, ctx); // a call: tab → server → worker +``` + +**Who the tab is** is not the tab's to say. The server authenticates the WebSocket when it connects (a cookie, the +owner on loopback, a guest's share link) and stamps that principal on everything the tab asks for. The worker's +`Notes` factory receives it and may give a guest a read-only variant. A tab cannot claim to be the owner. + +**What a tab serves** is only `local` types: state of this tab's UI, never shared. A draft that survives switching +panels, a selection, a tab-local cache: + +```ts +// plugins/notes/web.tsx: the web facet serves a local type, and contributes a panel +export default defineFacet((host, self) => { + host.serve(NoteDraft, (_params, self) => ({ text: self.state(DraftDoc) })); // lives in this tab only + self.contribute(Panels, { title: "Notes", Component: NotesPanel }); +}); +``` + +**When the connection drops** (a phone in a tunnel, a server restart), the tab keeps its references. Their values +stay readable, their `state` says `loading`, calls made meanwhile wait; on reconnect every reference is held again and +gets the current value in one frame. The UI shows a banner from `ref.state`, nothing more. + +**A terminal UI** is the same in Node: a client host connected to the server over a WebSocket, loading the plugins' +`tui` facets, rendering with whatever terminal library the app chose. **What a client never does**: serve a `remote` +type, be routed to, or keep data others rely on. Data that must outlive a tab belongs in an object on the server or a +worker. + +# Plugins {#plugins lede="Move the code into a directory that can be added, edited and removed while the app runs."} + +So far the facets were part of the app's own code. A **plugin** is the same code in its own directory: + +```text +plugins/notes/ + package.json { "name": "@app/notes" } + contract.ts the Notes contract (other plugins import it) + worker.ts the facet for workers: serves Notes + server.ts the facet for the server: an export route + web.tsx the facet for browser tabs: a notes panel +``` + +The plugin's name is its `package.json` name. Each file named after a **host kind** is that kind's facet, its +default export a `defineFacet(...)`. A plugin has only the files it needs: a plugin that only adds a route has a +`server.ts` and nothing else. + +```ts +// plugins/notes/worker.ts +import { defineFacet } from "@app/api/v1"; +import { Notes } from "./contract.ts"; + +export default defineFacet((host) => { + host.serve(Notes, ({ project }, principal, self) => { + const doc = self.state(NotesDoc); + return { variant: "editor", members: { list: doc, add: async ({ text }, ctx) => /* … */ } }; + }); +}); +``` + +`@app/api/v1` is your app's **API**, version 1: the part of its base code plugins import, as a host package (the +plugin's build leaves it out; every host passes its own copy in). It holds what plugins share and never reloads: +Chord's builders re-exported (`local`, `remote`, `obj`, `str`, ...), host kinds, the principal type, a `defineFacet` +bound to it. A plugin never imports Chord itself; the version in the path says what it was written against. + +# Loading Plugins {#loading lede="Two steps in two places: building a plugin into a script happens once, on the server; evaluating it happens in every process that runs it."} + +A plugin is source code on the server's disk: TypeScript files that import each other, import other plugins' +contracts, import npm packages, and import things the app itself provides. None of the processes that run the plugin +can use those files as they are. A browser tab has no file system and no TypeScript; a worker could read the files, +but then every worker would compile every plugin again, and two workers started a minute apart could see two different +versions. So loading a plugin is two separate steps: + +1. **Build**: turn each plugin facet (`notes/worker.ts`, `notes/web.tsx`) into one self-contained **script**. This + happens once per edit, on the server, for every host kind. +2. **Evaluate**: run a script inside one process, which gives back the facet, ready for `host.load`. This happens in + every process that hosts that kind: the server for its own facets, each worker for the worker facets, each tab for + the web facets. + +The two steps need different things, and that is why the API has two calls. + +## Host packages: what a script leaves out + +Look at what `notes/worker.ts` imports: + +```ts +import { defineFacet, obj, remote, Session, str } from "@app/api/v1"; // the app's API: its own part and Chord's +import { Notes } from "./contract.ts"; // the plugin's own file +import slugify from "slugify"; // an npm package in its dependencies +``` + +The last two go into the script: the plugin's own files and its npm dependencies are copied in. The first must not. +If the script carried its own copy of the API, and with it of Chord, the worker would run two Chords: the plugin's +`ChordError` would not be the worker's `ChordError`, and the plugin's `defineFacet` would make a facet the worker's host +does not recognize. The plugin must see the very `Session` host kind and the very `Panels` registry the process +already has, not a copy. + +Packages the process already has, and that every plugin must share, are its **host packages**. A script leaves them +out and asks for them by name when it runs. Building therefore needs only their **names**: "do not copy `@app/api/v1` +into this script". + +## Building: `createPlugins`, on the server + +```ts +const plugins = createPlugins({ + roots: [{ path: "./plugins", required: true }], // where the plugin directories are + api: { name: "@app/api", versions: ["v1"] }, // the app's API and the versions it provides + kinds: { + server: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/server"] }, + worker: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/worker"] }, + web: { platform: "browser", hostPackages: ["@app/api/v1", "@app/api/v1/web", "react", "react/jsx-runtime"] }, + }, + onError, +}, ctx); +``` + +`createPlugins` is the builder, and it builds for every kind, because only the server has the sources. For each kind +it needs two facts. The **platform** says where the scripts run: a `node` script may import `node:fs`, a `browser` +script may import a stylesheet but not `node:fs`. The **host package names** say what to leave out; they differ per +kind, because a tab also shares React with its plugins and a worker does not, and the API's per-kind modules +(`/v1/web`) belong to their kind only. With `api` set, the build checks every import of the API: a version the app +does not list, another kind's module, two versions in one facet, or Chord imported directly is a build error naming +the file and line. Nothing here runs a plugin. + +## Evaluating: `plugins.load`, in the process that hosts + +Evaluating is where the names become things. When the script runs and asks for `@app/api/v1`, someone has to hand +it the actual module, the one this process imported when it started. Only the process itself has it: + +```ts +// server.ts +import * as api from "./api/v1/index.ts"; // the app's API, the module plugins import as "@app/api/v1" +import * as apiServer from "./api/v1/server.ts"; +import { serverFacet } from "./core.ts"; + +const report = await plugins.load(server, { + kind: "server", // evaluate the server kind's scripts + modules: { "@app/api/v1": api, "@app/api/v1/server": apiServer }, // this process's own instances + core: [{ name: "@app/core", facet: serverFacet }], // base-code facets, loaded beside the plugins +}, ctx); +``` + +`modules` is this process's answer to every host package name. The server passes its own; a worker passes its own in +`workerMain`; a page passes its own in `startPage`. Each process has different instances, which is exactly the point: +inside each process, plugins share that process's Chord and base code. + +`core` is the app's base code in facet form. It is a facet like a plugin's, but it is not a plugin: it is made in the +process's own code, often from values only this process has (`core.workerFacet(start)` closes over a worker's start +data and store), and it never reloads, because it defines what all plugins share. Passing it here loads it together +with the plugins, so they are published in one step. + +## Why `plugins.load(host)`, not `host.load` + +A host has its own `load`, and `plugins.load` calls it. The host's `load` takes facets that already exist: functions +in memory. It knows nothing about directories, builds, scripts or modules, on purpose: the same host runs in a test +with facets imported directly, or in a Cloudflare Durable Object where plugins are bundled at deploy time and nothing +is built or evaluated at runtime. + +The plugins component is the part that knows about directories. It finds the plugins, builds them, evaluates them +with the process's modules, and then calls `host.load` with the result. It also remembers, per host, which facet each +plugin runs, so that a later reload can hand the host its complete new plugin set, with unchanged plugins as the very +same objects. That memory is why the host is passed to it: `plugins.load(server, …)` on the server, +`plugins.load(worker, …)` in a worker (where `workerMain` gives it a `plugins` that only evaluates), the same call +in every process. + +:::flow wide | Building on the server needs names; evaluating in each process needs that process's modules. +row "Server: build (once per edit)" accent +box src "plugin sources" "`notes/worker.ts`, imports `@app/api/v1`" muted +box build "createPlugins" "copies own files and npm deps; leaves host packages out" accent +box script "worker script" "`require(\"@app/api/v1\")` left as a name" +row "Each worker: evaluate" blue +box mods "the worker's modules" "its own `@app/api/v1`, its own Chord" muted +box load "plugins.load(worker, …)" "runs the script, answers each name" blue +box facet "the facet" "uses the worker's `Session`, the worker's `defineFacet`" +arrow src -> build +arrow build -> script +arrow script -> load "shipped" +arrow mods -> load +arrow load -> facet +::: + +## What happens, step by step + +:::steps | `plugins.load(server, { kind: "server", … })` ++++ Discover +The component lists `./plugins` and reads each `package.json`. Its `name` is the plugin's name; the files named after +a kind are its facets. A directory without a `package.json`, or with two `server` files, is reported, not loaded. Files +at the root's top named `..ts` (`hello.server.ts`) are single-file plugins, grouped by name. ++++ Build +For each plugin with a `server` facet it builds one script, if no current one exists. A syntax error or an import the +rules do not allow (a path into base code, a package not in the plugin's dependencies) is a failed build with the file +and line, never a crash. ++++ Evaluate +The script runs. Each host package name it asks for is answered from `modules`; anything else it may not have. The +script's default export must be a facet. ++++ Load +`host.load` runs the setups of `core` and of every plugin facet and publishes them together. A plugin whose build, +evaluation or setup failed is left out and listed in `report.failed` with the reason; the others load. Objects that +need it wait until a later edit loads it. +::: + +Only the server builds. Workers and clients (browser tabs, terminal UIs) only evaluate: the server builds their kinds +too, a worker gets its scripts in its start message, a tab or a TUI gets them from the server over HTTP, and each +evaluates them with its own modules (a worker's from `workerMain`, a page's from its base bundle). + +# Plugins Using Plugins {#together lede="Two ways one plugin uses another, and the one it may not."} + +A search plugin wants to search the notes. It **imports the contract** and asks for the object: + +```ts +// plugins/search/worker.ts +import { Notes } from "../notes/contract.ts"; // a plugin of the same root, by relative path: copied into search's script + +host.serve(Search, ({ project }, principal, self) => { + const notes = self.get(Notes, { project }); // a dependency: Search goes live once Notes is + const index = self.derive([notes.list], ([list]) => buildIndex(list.items)); + return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } }; +}); +``` + +Which file a plugin imports decides which version of the contract it claims, so the rule is: **a plugin carries the +contract it was written against.** Two cases: + +- **Plugins that change together** (the app's own, in its repository; a user's, in one root) import each other by + relative path, `../notes/contract.ts`: the same commit holds both sides. The file is copied into the importer's + script, so only contracts and other side-effect-free files belong there (a copied file may import only the API and + files of its own plugin). A relative path leaving the root (into base code, into another root) is a build error. +- **Plugins from another repository** take contracts as a dependency at a fixed version, by package name: + `@bob/notes` (its `contract.ts`, through the package's `exports`), `@bob/notes/web` (its `contract.web.tsx`), pinned + in `package.json` by an npm version or a git ref (`"@bob/notes": "github:bob/notes#"`). The build inlines + the installed version: when Bob's notes move to contract version 2, a consumer built against version 1 keeps + claiming version 1, and the provider serves it through `accepts` or the consumer waits "contract mismatch", visible, + never silently wrong. A plugin inside a repository holding several (which lists them in its own `package.json`, + `"chord": { "plugins": ["./notes", "./search"] }`) is a dependency on the whole repository, imported by path into it: + `@bob/plugins/notes/contract.ts`. + +A contract file may be for one kind: `contract.web.tsx` (React components in a type, a panel's registry token) is a +**web file**. Importing it, directly or through other files, from a facet of another kind is a build error naming +the chain (`server.ts → view.ts → ../notes/contract.web.tsx`). The shared `contract.ts` mentions native things only +through `import type`. + +`self.get` in a factory declares a dependency: Chord makes `Search` live only once `Notes` is, and if the notes +plugin is missing, `Search` waits with a clear reason ("waiting for notes: not served here") instead of failing. + +The other way is a **registry**: an extension point one plugin defines and others contribute to. The app's API +defines `Panels` in its web module, and every plugin with a UI contributes one: + +```tsx +// plugins/notes/web.tsx +import { Panels } from "@app/api/v1/web"; +export default defineFacet((host, self) => self.contribute(Panels, { title: "Notes", Component: NotesPanel })); +``` + +What a plugin may not do is reach into another's data. Search never reads notes' documents; it uses the `Notes` +object's state. That rule is what lets the notes plugin change its storage without breaking search. + +# Changing a Plugin {#reload lede="Edit, build, roll out: how new code takes over while every tab stays connected."} + +You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. +You (or an agent) build and roll out: + +```ts +const built = await plugins.build(plugins.state.value.dirty, ctx); +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx); +``` + +The rollout goes host by host, server first, then every running worker. On each host Chord compares the new plugin +set with the loaded one, runs the new factories against a recording `self` to see what each object would declare, and +then gives every object of the changed plugin its fate: + +| The object | What happens | +| --- | --- | +| holds no natives (open files, sockets), the new code neither | **swapped** in place: new methods, same documents; running calls finish on the old code | +| holds natives | **restarted**: calls wait, running ones finish, natives close and reopen with the new code | +| its type was removed | **ended**; its users wait until something serves it again | + +The tab's reference follows the new object and gets one frame with the difference. Nothing reconnects. If the edit +is broken (a syntax error, a factory that throws), that plugin keeps its old code on that host and the report says why; +every other plugin of the rollout applies. + +:::note | Versions +A change to a contract (a new field in `Notes`'s state) gets a new `version`. A tab built with the old contract keeps +working if the provider maps the old version (`accepts`); otherwise it waits with "version mismatch" until it reloads. +::: + +# Base Code and Plugins {#base lede="What goes into the app itself, what goes into plugins, and how an app ships plugins of its own."} + +Every app has two kinds of code, and the line between them decides what can change while the app runs. + +**Base code** is the app's own program: `src/`, started by `node server.ts`. It shrinks to three things: the **API** +(the versioned modules plugins import), the **bootstrap** (creating hosts, opening the store, components such as +`createHttp` and `createWorkers`) and one small **start-data facet** serving process values (start data, the opened +store) as core objects. Base code changes only with a restart: if it reloaded, the process would hold two `Panels` +registries, and old and new plugins would no longer meet. + +**Plugins** are everything else: notes, search, export, a terminal, the harness, the providers of the app's core +registries. They reload on an edit, a broken one is left out alone, and an agent can write one while the app runs. + +## The API, by version and by kind + +Plugins import their host only through the API, a version in the path: + +```ts +// src/api/v1/index.ts: "@app/api/v1", for every kind +export { defineRegistry, local, obj, remote, str } from "@earendil-works/chord/next"; // the Chord surface the app supports +export { defineFacet, Project, Server } from "../../core.ts"; // bound to the app's principal +// src/api/v1/web.ts: "@app/api/v1/web", for pages only +export const Panels = defineRegistry("app.panels", { version: 1 }); +``` + +The app lists the versions it provides, newest first (`api: { name: "@app/api", versions: ["v2", "v1"] }`), and each +kind's modules in `kinds..hostPackages`. When the app changes its API in a way that breaks plugins, it adds +`/v2` and keeps `/v1` as a thin layer over it for a while; once `/v1` goes, a plugin importing it fails to build with +"@app/api/v1 is not available: this app provides v2", the plugin alone, the message saying what to change. One facet +uses one version. Contracts the app promises others are re-exported in the API (`export { Notes } from +"../../plugins/notes/contract.ts"`), so a plugin built against `/v1` keeps v1's contract files. + +## Plugins the app ships + +An app comes with plugins of its own. They follow the same rules as anyone's, in a root inside the app, next to the +root where users and agents add theirs: + +```ts +const plugins = createPlugins({ + roots: [{ path: "./plugins", required: true }, join(dataDir, "plugins")], // the app's own, then the user's + api: { name: "@app/api", versions: ["v1"] }, + kinds, + onError, +}, ctx); +``` + +They are built at startup like any plugin (a quarter of a second for twenty), reload like any plugin, and are left out +alone when broken at a reload. The app's root is **required**: a plugin of it failing at startup makes +`plugins.load` reject with a `PluginError` naming it (after the load settled), so the app stops instead of running +without its core. A failure in the user's root is reported and the app runs on. If two roots have a plugin with the +same name, the first root wins, and the other is reported. + +## Small plugins, installed plugins, tested plugins + +- **A single file** at a root's top is a plugin: `plugins/hello.server.ts` and `plugins/hello.web.tsx` are the plugin + `hello` with two facets. It has no `package.json` and imports only the API; when it needs a second file or a + package, it becomes the directory `hello/`, which a reload treats as the same plugin. +- **Installing** from git or npm: `plugins.installPlugin({ git: url, ref }, root, ctx)` clones the ref (a branch, tag + or commit) into the root, `{ npm: name, version }` extracts the package; then its dependencies are installed. Removal + is deleting the directory. +- **Testing** a plugin alone: `testPlugin("./plugins/stats", { kind: "worker", api, modules, stubs }, ctx)` builds it + as production does, loads it on an in-memory host beside stub facets serving what it uses, collects what it + contributes to registries nobody serves, and lists the contracts it waits for (`docs/next/plugin-authoring.md`). + +# The App Kit {#kit lede="startServer and startWorker: the hosts, stores, sign-in, files and plugin roots every app needs, so an app is its API, its plugins and a start script."} + +The chapters so far wired an app by hand: a store, a host, `createHttp`, `createWorkers`, `createPlugins`, an +authenticator, a worker entry connecting to its parent. Every app needs the same wiring, and most of it has one right +answer. The **app kit** (`chord/next/app`) is that answer, opinionated on purpose: one data directory layout, one way +to sign in, one place for files. + +## A whole app + +The notes app of example 53 is four files of base code and two plugins: + +```ts +// start.ts +const app = await startServer({ + name: "notes", // the data directory: ~/.notes unless dataDir says otherwise + root: "./plugins", // the app's own plugins: a required root + api: { name: "@notes/app", versions: ["v1"], modules: { "@notes/app/v1": v1, "@notes/app/v1/server": v1server } }, + kinds: { server: { platform: "node" }, worker: { platform: "node" } }, + principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", project: address.key }) }, + workers: [{ + kind: Project, // one worker per project + entry: "./worker.ts", + start: (address) => ({ project: address.key }), + authorize: (principal) => principal.kind === "owner" || principal.kind === "server", + }], + onError: (error) => log(error), +}, ctx); +console.log(`Open ${app.ownerLink()}`); + +// worker.ts +startWorker({ api: API, kinds: KINDS, onError: (error) => log(error) }); +``` + +`startServer` creates the data directory (`server/store.sqlite`, `workers//

/`, `plugins/`, `logs/`, +`owner-token`), opens the server's store, creates the server host, HTTP, a workers component per worker kind and the +plugins component over two roots: the app's `plugins/` and the user's `/plugins/`. Then it loads the server's +plugins with the kit's own facets beside them and resolves. `startWorker` does the worker's half: the project's +directory and store, the host at the project's address, the link to the server first, the worker's plugins. + +## What plugins get from the kit + +Plugins still import only the app's API; the API re-exports the kit's objects it offers: + +- `AppInfo` on the server (data directory, app name, API version) and `WorkerInfo` on workers (the address and the + start data `start(address)` made): the **start-data objects**. Process values reach plugins only through them. +- `Files`, one per host: `put` for small files, `begin`, `append` and `finish` for large ones, ranged `read`, `remove` + and a watchable `list`, stored in the host's directory with caps per host. +- `Sessions` on the server: `issue`, `revoke`, `list`. + +This finishes what the previous chapter started. Base code is the API, the start script and the worker's entry. The +providers of the app's own registries live in its required root: example 53's API declares a `Sections` registry, and +its home plugin serves it and reads `AppInfo`, so it reloads like any other plugin. + +## Who may come in + +HTTP **denies by default**: a request without a principal gets 401 before any plugin code runs, unless the route says +`public: true`. Someone gets a principal through a **session**: a random token, kept in the store only as a hash, sent +as a signed cookie by a browser or as `Authorization: Bearer` by a command-line client. + +The first session is the owner's. At each start the kit makes a one-time **owner link**, +`http://127.0.0.1:8788/_chord/owner#`, valid ten minutes; the start script prints it. The token is in the +fragment, which browsers never send, so it never reaches a log: the page posts it, the kit consumes it and sets the +owner's cookie. A command-line client on the same machine reads `/owner-token` (mode 0600). One on another +machine **pairs**: it asks for a code, the owner approves the code in a signed-in browser, and the client's next poll +returns its token (example 54). + +Users other than the owner come from optional plugins the app adds: `defineUsers(profile)` for accounts, +`inviteFacet()` for invite links, `ephemeralLoginFacet()` for "pick a name and you're in". + +## Applying changes, moving projects + +The kit never reloads by itself. An edit or an install marks plugins dirty; `app.plugins.state` drives the app's +"apply changes" button, and `app.applyChanges(ctx)` builds and rolls out to the server, then the workers. + +A project's data is what its worker created: its store and its files. `app.exportData(Project({ id }), path, ctx)` +writes them as one archive (a manifest, the store's export, the files) and stops nothing: a running worker writes its +own store at one consistent point. `app.importData(path, Project({ id: "copy" }), ctx)` makes a new project from it and +refuses one that already has data (example 55). + +# Reference {.appendix} + +| Call | What it does | +| --- | --- | +| `createHost

({ address?, principal?, store?, onError })` | a host: addressable with an address, else a client | +| `host.load([{ name, facet }], ctx)` / `host.reload(...)` | publish facets / apply a new plugin set | +| `host.get(Type, params, ctx)` / `host.hold(...)` | a live reference / a reference at once that follows its object | +| `local(name, { version, params, members })` / `remote(name, { version, params, hostAddress, variants })` | contracts | +| `defineFacet((host, self) => …)`, `host.serve(Type, factory)`, `self.contribute(R, value)` | a facet | +| factory `self`: `state`, `derive`, `get`, `registry`, `open`, `on`, `onLive`, `job`, `tx` | an object's parts | +| `createPlugins({ roots, api, kinds, onError }, ctx)` | the plugins component | +| `plugins.load(host, { kind, core, modules }, ctx)`, `build(names)`, `rollout(steps, built)` | load, build, roll out | +| `plugins.installPlugin(source, root, ctx)`, `testPlugin(dir, options, ctx)` | install a plugin, test one alone | +| `hostKind(name, params?)`, `Kind(params)` | host kinds and addresses | +| `startServer

({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, onError }, ctx)` | an app's server, with the kit | +| `startWorker({ api, kinds, busy?, onError })` | a worker's entry, with the kit | +| `app.ownerLink()`, `app.applyChanges(ctx)`, `app.plugins.install(source, ctx)` | sign in, apply changes, install | +| `app.exportData(address, path, ctx)`, `app.importData(path, address, ctx)` | move a worker's data | +| `AppInfo`, `WorkerInfo`, `Files`, `Sessions` | the kit's objects plugins get | diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html new file mode 100644 index 000000000..d0ec02a3c --- /dev/null +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -0,0 +1,781 @@ + + +Hosts and Plugins: a tutorial for Chord v2 + + + + +

+
+
A guide · Chord v2 · 2026-11-02
+

Hosts
and Plugins

+

Build a small notes app step by step, from one host to plugins on a server and on workers, edited while it runs

+
3kinds of host in an app: server, worker, client
1file per host kind in a plugin
0restarts to apply a plugin edit
1rule: plugins talk only through object types and registries
+

A tutorial for people building apps on Chord v2, written by the redesign session from design.md, slice-7c.md, +slice-10.md, slice-12.md, slice-13.md, the decision log and the code in packages/chord/src/next and +src/next-node. API names are the current ones. The early chapters wire hosts by hand to show the parts; the last +chapter shows the app kit (chord/next/app), which does that wiring for an app.

+
+
+ + +
+

Before we begin

+

You want to build a notes app. You type a note in a browser tab, and every other tab sees it at once. Each project +gets its own notes in its own process, so a project that misbehaves cannot take the others down. And you, or an agent +you work with, add features while the app runs: a search, an export button, a word count, without restarting +anything.

+

Chord gives you three things for that. Hosts: the processes of your app (the server, a worker per project, the +browser tabs), connected to each other. Objects: the state and methods your app is made of, living on one host and +used from any other. Plugins: directories of code that declare objects for each kind of host, loaded and +reloaded while the app runs.

+

This book builds the app in that order. Each chapter adds one concept and shows complete code.

+

Seven things to know

+
    +
  1. A host is one participant. The server, each worker, each browser tab and each terminal UI is a host; hosts +connect over links. Tabs and terminal UIs are clients: no address, they connect to the server and ask.
    2. +
  2. An object has a type, and a type is a contract. The contract names the object's state and methods; any host can +ask for an object by type and params, and Chord finds where it lives.
    3. +
  3. A facet declares, an object acts. A facet is a plugin's code for one host kind: it says which object types this +host serves. The objects hold the state and do the work.
    4. +
  4. A plugin is a directory (or a single file). A package.json with the plugin's name, and one file per host kind: +server.ts, worker.ts, web.tsx. It imports its host only through the app's versioned API (@app/api/v1).
    5. +
  5. The server builds, every host evaluates. The plugins component builds each facet into a small script; the +server, workers and tabs load the scripts for their kind.
    6. +
  6. Plugins meet through types and registries. One plugin uses another's objects through its contract, or +contributes to a registry another defines. Nothing else crosses between plugins.
    7. +
  7. An edit is a rollout. Change a file, build, roll out: the server, then every worker, swap the plugin's objects in +place; a broken edit keeps the old code running and reports why.
    8. +
+
ON DISKSERVER PROCESSCLIENTS AND WORKERSplugins/notes/server.ts`, `worker.ts`,`web.tsx`, `tui.ts`, `contract.tssrc/ (base code)`@app/api/v1`: Chord's surface, hostkinds, principalsplugins componentdiscover, build every facet, watch,roll outserver hostserver facets; routes Project →workersclient: tab or TUIweb or tui facets, local UI objects;holds `Notes { p1 }`worker p1worker facets; serves `Notes { p1 }`read, buildhost packagesserver scriptsweb or tui scriptshold Notes { p1 }start (worker scripts), relayEach host loads only its own kind's facets. Plugins meet only through object types and registries.
Figure 1. How it all fits: plugins on disk are built once on the server; each host evaluates the facets of its kind; tabs and the TUI are clients that reach workers' objects through the server.
+

Words used throughout

+

A host is one participant of the app, made with createHost. A host kind (Server, Session) describes +addressable hosts; an address (Session({ id: "p1" })) names one. An object type is a contract: local (used +on its own host only) or remote (used from any host). An object is one instance of a type, per params. A +facet is a plugin's declarations for one host kind. A plugin is a directory with a package.json. A +reference is what you get when you ask for an object (host.get, self.get, ctx.get). A document is data an object +stores, changed in transactions; a table holds many records with queries, in the host's store; a derive is a value computed from others; a native is an outside +resource an object opens and closes. A registry is an +extension point: one plugin defines it, others contribute. A rollout applies a build to the hosts in order.

+
+ +
I

One Host

The smallest app: one process, one object type, one call.

+

Start without plugins. One host, one object that keeps notes:

+
import { createHost, defineFacet, defineState, local, method, obj, arr, str } from "@earendil-works/chord/next";
+
+const NoteList = obj({ items: arr(str()) });          // one schema, used by the contract and the document
+
+// the contract: what a Notes object offers
+const Notes = local("notes.store", {
+	version: 1,
+	params: {},                                        // one object per host: no params
+	members: {
+		list: NoteList,                               // state anyone can read and watch
+		add: method(obj({ text: str() })),            // a method: arguments checked by Chord
+	},
+});
+
+// the document the object stores (its own version: storage can change without changing the contract)
+const NotesDoc = defineState("notes", { schema: NoteList, version: 1, initial: () => ({ items: [] }) });
+
+// the facet: which types this host serves, and how to make each object
+const facet = defineFacet((host) => {
+	host.serve(Notes, (_params, self) => {
+		const doc = self.state(NotesDoc);
+		return {
+			list: doc,
+			add: async ({ text }, ctx) => self.tx(async (tx) => { (await tx.draft(doc)).items.push(text); }, ctx),
+		};
+	});
+});
+
+const server = createHost({ onError: console.error });
+await server.load([{ name: "@app/notes", facet }], ctx);
+const notes = await server.get(Notes, {}, ctx);        // a reference: live, with its state
+await notes.add({ text: "buy milk" }, ctx);
+console.log(notes.list.value);                          // { items: ["buy milk"] }
+

Four things happened. The contract Notes says what the object offers; the document says what it stores. +Here they have the same shape, so one schema serves both; they are separate because they change separately (an +object may offer a computed value or a shortened view instead of what it stores, and storage can change its shape +without users noticing). +The facet says this host serves +Notes, with a factory that declares the object's parts: a document, and a method writing it in a transaction. +load publishes the facet. get returns a reference once the object is live; its members are the contract's.

+

The factory only declares: it is synchronous and opens nothing. Chord activates the object (loads its documents, opens +what it asked to open) and then serves calls. That is why the same factory can run again when the code changes.

+
+ +
II

An Object's Parts

Documents, tables, transactions, computed values, watching, and outside resources: the vocabulary of every factory.

+

A factory builds an object from a handful of parts, each declared with one call on self. The notes object grows to +show all of them.

+

Documents hold the object's data: self.state(NotesDoc) gives a handle whose .value is the current data. Data +changes only in a transaction: self.tx(fn, ctx) runs fn with tx, and await tx.draft(handle) gives a +mutable copy of the document's data; whatever you change on it is committed at once when fn returns, or not at all +if it throws.

+
add: async ({ text }, ctx) => self.tx(async (tx) => {
+	const draft = await tx.draft(doc);          // the document's data, writable inside this transaction only
+	draft.items.push(text);                      // recorded as a change, committed when the function returns
+}, ctx),
+

Documents live in memory unless they are persisted: defineState(..., { persist: true, checkpointWhen }) keeps +one in the host's store (a SQLite file the app opens and gives to createHost({ store })), loaded when the +object activates. For many records with queries, an object declares a table:

+
const NoteRows = defineTable("notes", {
+	version: 1,
+	columns: { id: str(), text: str(), at: int() },
+	key: ["id"],
+	indexes: { by_time: ["at"] },
+});
+
+host.serve(NoteStore, (_params, self) => {               // a local type with empty params: it may persist
+	const rows = self.table(NoteRows);                    // created (or migrated) when the object activates
+	const recent = query(NoteRows).orderBy((n) => n.at, "desc");
+	return {
+		add: async ({ text }, ctx) => self.tx(async (tx) => tx.put(rows, { id: uuid(), text, at: Date.now() }), ctx),
+		page: async ({ after }, ctx) => rows.page(recent, 50, ctx, { after }),   // 50 at a time, with a cursor
+	};
+});
+

Persisted data lives only in local objects with empty params (one per host and type), so its address never depends +on who asked or with which params; an object with params (a view per project) reads from such a store. Writes go +through the same transactions as documents: a note row and a document can change in one commit.

+

A derive is a value computed from other values, recomputed whenever they change. It is how an object offers +something it does not store:

+
const count = self.derive([doc], ([notes]) => notes.items.length);   // 0, 1, 2, … as notes are added
+return { list: doc, count, add };                                      // `count` can be a member like `list`
+

Watching is how anyone follows a value as it changes: a document handle, a derive, a reference's member. watch +gives the current value, then one call per change:

+
const watch = await notes.count.watch(ctx);
+watch.start(async (call) => {
+	if (call.kind === "value") console.log("notes:", call.value);   // "failed" calls say a value cannot be computed now
+});
+

A routine is the object watching on its own behalf: self.on(source, fn) runs fn with each new value, for as +long as the object lives (a reconciler: "whenever the notes change, rebuild the export file").

+

A native is something outside Chord the object opens and must close: a file watcher, a database connection, a +child process. self.open(open, close) opens it while the object activates and closes it when the object ends:

+
const inbox = self.open(async (ctx) => watchDirectory(inboxPath, ctx.abortSignal), (watcher) => watcher.close());
+self.onLive(async (ctx) => {                                  // a routine started once the object is live
+	for await (const file of inbox.value.files(ctx.abortSignal)) await importNote(file, ctx);
+});
+

Natives are why some objects restart when their plugin changes: an open connection cannot be moved to new code, so +Chord closes it and opens it again. Objects without natives are swapped in place.

+
+ +
III

Several Hosts

A server, a worker per project, browser tabs: how they find each other's objects.

+

Now give each project its own process. Declare the kinds of addressable host, and make notes a remote type that +lives on its project's worker:

+
export const Server = hostKind("server");
+export const Project = hostKind("project", { id: str() });
+
+export const Notes = remote("notes", {
+	version: 1,
+	params: { project: str() },
+	hostAddress: ({ project }) => Project({ id: project }),   // where an object of these params lives
+	variants: { editor: { state: { list: NoteList }, methods: { add: method(obj({ text: str() })) } } },
+});
+

Two things are new. A remote type can be used from any host, and it has variants: different sets of members +for different callers, chosen by the factory (an editor variant with add, a reader variant for guests without +it). And hostAddress says where its objects live; it is the whole routing story.

+

A tab asks for Notes { project: "p1" }. Its host computes the address, +Project(p1), and sends the request to its parent, the server. The server has a route for Project: it starts (or +finds) the worker for p1 and passes the request on. The worker serves Notes. The tab gets a reference whose state +follows the worker's document, one frame per change.

+
CLIENTSSERVERWORKERStab Aholds Notes { project: p1 }tab Bholds Notes { project: p1 }serverroute Project → workerworker p1serves Notes { p1 }worker p2serves Notes { p2 }holdholdrelay
Figure 2. A tab holds a project's notes: the server routes by address to the project's worker and relays, without reading the data.
+

Each host is created the same way, with what it is:

+
// server.ts
+const server = createHost<P>({ address: Server(), principal: { kind: "server" }, onError });
+
+// worker.ts (started by the server for one project)
+const worker = createHost<P>({ address: Project({ id }), principal: { kind: "worker", project: id }, onError });
+worker.connect(parentLink, { principal: { kind: "server" }, trusted: true }, ctx);
+
+// a browser tab: no address, it is a client
+const tab = createHost<P>({ onError });
+tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx);
+

P is your app's type of principal, who is calling: an owner, a guest, the server, a worker. Links stamp it; +a remote object is made per principal, so a guest can get a read-only variant of the same notes.

+
The app kit does this wiring

Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components +(createWorkers, chord.http). An app built with the app kit (chord/next/app, the last chapter) does not wire them: +startServer(...) on the server and startWorker(...) in a worker's entry do.

+
+
+ +
IV

Client Hosts

Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot.

+

A browser tab and a terminal UI are hosts like the server, with one difference: they have no address. Nobody can +send a request to a tab; a tab only connects to the server and asks for things. That is a deliberate rule: a tab +runs on a device the server does not control, so nothing on the server ever waits for, or trusts, a tab's state.

+
// a browser tab (slice 11's page host will do this wiring for you)
+const tab = createHost<P>({ onError });                               // no address, no principal: a client
+tab.connect(webSocketLink(new WebSocket("/chord")), { principal: { kind: "server" } }, ctx);
+await tab.load([{ name: "@app/web", facet: webCore }, ...webFacets], ctx);
+
+const notes = tab.hold(Notes, { project: "p1" }, ctx);               // a reference at once, live soon
+(await notes.list.watch(ctx)).start(async (call) => {
+	if (call.kind === "value") render(call.value.items);              // every change, as it commits on the worker
+});
+await notes.add({ text: "from the tab" }, ctx);                        // a call: tab → server → worker
+

Who the tab is is not the tab's to say. The server authenticates the WebSocket when it connects (a cookie, the +owner on loopback, a guest's share link) and stamps that principal on everything the tab asks for. The worker's +Notes factory receives it and may give a guest a read-only variant. A tab cannot claim to be the owner.

+

What a tab serves is only local types: state of this tab's UI, never shared. A draft that survives switching +panels, a selection, a tab-local cache:

+
// plugins/notes/web.tsx: the web facet serves a local type, and contributes a panel
+export default defineFacet((host, self) => {
+	host.serve(NoteDraft, (_params, self) => ({ text: self.state(DraftDoc) }));   // lives in this tab only
+	self.contribute(Panels, { title: "Notes", Component: NotesPanel });
+});
+

When the connection drops (a phone in a tunnel, a server restart), the tab keeps its references. Their values +stay readable, their state says loading, calls made meanwhile wait; on reconnect every reference is held again and +gets the current value in one frame. The UI shows a banner from ref.state, nothing more.

+

A terminal UI is the same in Node: a client host connected to the server over a WebSocket, loading the plugins' +tui facets, rendering with whatever terminal library the app chose. What a client never does: serve a remote +type, be routed to, or keep data others rely on. Data that must outlive a tab belongs in an object on the server or a +worker.

+
+ +
V

Plugins

Move the code into a directory that can be added, edited and removed while the app runs.

+

So far the facets were part of the app's own code. A plugin is the same code in its own directory:

+
plugins/notes/
+  package.json        { "name": "@app/notes" }
+  contract.ts         the Notes contract (other plugins import it)
+  worker.ts           the facet for workers: serves Notes
+  server.ts           the facet for the server: an export route
+  web.tsx             the facet for browser tabs: a notes panel
+

The plugin's name is its package.json name. Each file named after a host kind is that kind's facet, its +default export a defineFacet(...). A plugin has only the files it needs: a plugin that only adds a route has a +server.ts and nothing else.

+
// plugins/notes/worker.ts
+import { defineFacet } from "@app/api/v1";
+import { Notes } from "./contract.ts";
+
+export default defineFacet((host) => {
+	host.serve(Notes, ({ project }, principal, self) => {
+		const doc = self.state(NotesDoc);
+		return { variant: "editor", members: { list: doc, add: async ({ text }, ctx) => /* … */ } };
+	});
+});
+

@app/api/v1 is your app's API, version 1: the part of its base code plugins import, as a host package (the +plugin's build leaves it out; every host passes its own copy in). It holds what plugins share and never reloads: +Chord's builders re-exported (local, remote, obj, str, ...), host kinds, the principal type, a defineFacet +bound to it. A plugin never imports Chord itself; the version in the path says what it was written against.

+
+ +
VI

Loading Plugins

Two steps in two places: building a plugin into a script happens once, on the server; evaluating it happens in every process that runs it.

+

A plugin is source code on the server's disk: TypeScript files that import each other, import other plugins' +contracts, import npm packages, and import things the app itself provides. None of the processes that run the plugin +can use those files as they are. A browser tab has no file system and no TypeScript; a worker could read the files, +but then every worker would compile every plugin again, and two workers started a minute apart could see two different +versions. So loading a plugin is two separate steps:

+
    +
  1. Build: turn each plugin facet (notes/worker.ts, notes/web.tsx) into one self-contained script. This +happens once per edit, on the server, for every host kind.
    2. +
  2. Evaluate: run a script inside one process, which gives back the facet, ready for host.load. This happens in +every process that hosts that kind: the server for its own facets, each worker for the worker facets, each tab for +the web facets.
    3. +
+

The two steps need different things, and that is why the API has two calls.

+

Host packages: what a script leaves out

+

Look at what notes/worker.ts imports:

+
import { defineFacet, obj, remote, Session, str } from "@app/api/v1";   // the app's API: its own part and Chord's
+import { Notes } from "./contract.ts";                                  // the plugin's own file
+import slugify from "slugify";                                          // an npm package in its dependencies
+

The last two go into the script: the plugin's own files and its npm dependencies are copied in. The first must not. +If the script carried its own copy of the API, and with it of Chord, the worker would run two Chords: the plugin's +ChordError would not be the worker's ChordError, and the plugin's defineFacet would make a facet the worker's host +does not recognize. The plugin must see the very Session host kind and the very Panels registry the process +already has, not a copy.

+

Packages the process already has, and that every plugin must share, are its host packages. A script leaves them +out and asks for them by name when it runs. Building therefore needs only their names: "do not copy @app/api/v1 +into this script".

+

Building: createPlugins, on the server

+
const plugins = createPlugins({
+	roots: [{ path: "./plugins", required: true }],      // where the plugin directories are
+	api: { name: "@app/api", versions: ["v1"] },          // the app's API and the versions it provides
+	kinds: {
+		server: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/server"] },
+		worker: { platform: "node", hostPackages: ["@app/api/v1", "@app/api/v1/worker"] },
+		web: { platform: "browser", hostPackages: ["@app/api/v1", "@app/api/v1/web", "react", "react/jsx-runtime"] },
+	},
+	onError,
+}, ctx);
+

createPlugins is the builder, and it builds for every kind, because only the server has the sources. For each kind +it needs two facts. The platform says where the scripts run: a node script may import node:fs, a browser +script may import a stylesheet but not node:fs. The host package names say what to leave out; they differ per +kind, because a tab also shares React with its plugins and a worker does not, and the API's per-kind modules +(/v1/web) belong to their kind only. With api set, the build checks every import of the API: a version the app +does not list, another kind's module, two versions in one facet, or Chord imported directly is a build error naming +the file and line. Nothing here runs a plugin.

+

Evaluating: plugins.load, in the process that hosts

+

Evaluating is where the names become things. When the script runs and asks for @app/api/v1, someone has to hand +it the actual module, the one this process imported when it started. Only the process itself has it:

+
// server.ts
+import * as api from "./api/v1/index.ts";                 // the app's API, the module plugins import as "@app/api/v1"
+import * as apiServer from "./api/v1/server.ts";
+import { serverFacet } from "./core.ts";
+
+const report = await plugins.load(server, {
+	kind: "server",                                          // evaluate the server kind's scripts
+	modules: { "@app/api/v1": api, "@app/api/v1/server": apiServer },   // this process's own instances
+	core: [{ name: "@app/core", facet: serverFacet }],       // base-code facets, loaded beside the plugins
+}, ctx);
+

modules is this process's answer to every host package name. The server passes its own; a worker passes its own in +workerMain; a page passes its own in startPage. Each process has different instances, which is exactly the point: +inside each process, plugins share that process's Chord and base code.

+

core is the app's base code in facet form. It is a facet like a plugin's, but it is not a plugin: it is made in the +process's own code, often from values only this process has (core.workerFacet(start) closes over a worker's start +data and store), and it never reloads, because it defines what all plugins share. Passing it here loads it together +with the plugins, so they are published in one step.

+

Why plugins.load(host), not host.load

+

A host has its own load, and plugins.load calls it. The host's load takes facets that already exist: functions +in memory. It knows nothing about directories, builds, scripts or modules, on purpose: the same host runs in a test +with facets imported directly, or in a Cloudflare Durable Object where plugins are bundled at deploy time and nothing +is built or evaluated at runtime.

+

The plugins component is the part that knows about directories. It finds the plugins, builds them, evaluates them +with the process's modules, and then calls host.load with the result. It also remembers, per host, which facet each +plugin runs, so that a later reload can hand the host its complete new plugin set, with unchanged plugins as the very +same objects. That memory is why the host is passed to it: plugins.load(server, …) on the server, +plugins.load(worker, …) in a worker (where workerMain gives it a plugins that only evaluates), the same call +in every process.

+
SERVER: BUILD (ONCE PER EDIT)EACH WORKER: EVALUATEplugin sourcesnotes/worker.ts`, imports`@app/api/v1createPluginscopies own files and npm deps; leaveshost packages outworker script`require("@app/api/v1")` left as anamethe worker's modulesits own `@app/api/v1`, its own Chordplugins.load(worker, …)runs the script, answers each namethe facetuses the worker's `Session`, theworker's `defineFacet`shipped
Figure 3. Building on the server needs names; evaluating in each process needs that process's modules.
+

What happens, step by step

+
plugins.load(server, { kind: "server", … })

Discover

The component lists ./plugins and reads each package.json. Its name is the plugin's name; the files named after +a kind are its facets. A directory without a package.json, or with two server files, is reported, not loaded. Files +at the root's top named <name>.<kind>.ts (hello.server.ts) are single-file plugins, grouped by name.

+

Build

For each plugin with a server facet it builds one script, if no current one exists. A syntax error or an import the +rules do not allow (a path into base code, a package not in the plugin's dependencies) is a failed build with the file +and line, never a crash.

+

Evaluate

The script runs. Each host package name it asks for is answered from modules; anything else it may not have. The +script's default export must be a facet.

+

Load

host.load runs the setups of core and of every plugin facet and publishes them together. A plugin whose build, +evaluation or setup failed is left out and listed in report.failed with the reason; the others load. Objects that +need it wait until a later edit loads it.

+
+

Only the server builds. Workers and clients (browser tabs, terminal UIs) only evaluate: the server builds their kinds +too, a worker gets its scripts in its start message, a tab or a TUI gets them from the server over HTTP, and each +evaluates them with its own modules (a worker's from workerMain, a page's from its base bundle).

+
+ +
VII

Plugins Using Plugins

Two ways one plugin uses another, and the one it may not.

+

A search plugin wants to search the notes. It imports the contract and asks for the object:

+
// plugins/search/worker.ts
+import { Notes } from "../notes/contract.ts";   // a plugin of the same root, by relative path: copied into search's script
+
+host.serve(Search, ({ project }, principal, self) => {
+	const notes = self.get(Notes, { project });   // a dependency: Search goes live once Notes is
+	const index = self.derive([notes.list], ([list]) => buildIndex(list.items));
+	return { variant: "main", members: { find: async ({ q }) => lookup(index.value, q) } };
+});
+

Which file a plugin imports decides which version of the contract it claims, so the rule is: a plugin carries the +contract it was written against. Two cases:

+
    +
  • Plugins that change together (the app's own, in its repository; a user's, in one root) import each other by +relative path, ../notes/contract.ts: the same commit holds both sides. The file is copied into the importer's +script, so only contracts and other side-effect-free files belong there (a copied file may import only the API and +files of its own plugin). A relative path leaving the root (into base code, into another root) is a build error.
    • +
  • Plugins from another repository take contracts as a dependency at a fixed version, by package name: +@bob/notes (its contract.ts, through the package's exports), @bob/notes/web (its contract.web.tsx), pinned +in package.json by an npm version or a git ref ("@bob/notes": "github:bob/notes#<commit>"). The build inlines +the installed version: when Bob's notes move to contract version 2, a consumer built against version 1 keeps +claiming version 1, and the provider serves it through accepts or the consumer waits "contract mismatch", visible, +never silently wrong. A plugin inside a repository holding several (which lists them in its own package.json, +"chord": { "plugins": ["./notes", "./search"] }) is a dependency on the whole repository, imported by path into it: +@bob/plugins/notes/contract.ts.
    • +
+

A contract file may be for one kind: contract.web.tsx (React components in a type, a panel's registry token) is a +web file. Importing it, directly or through other files, from a facet of another kind is a build error naming +the chain (server.ts → view.ts → ../notes/contract.web.tsx). The shared contract.ts mentions native things only +through import type.

+

self.get in a factory declares a dependency: Chord makes Search live only once Notes is, and if the notes +plugin is missing, Search waits with a clear reason ("waiting for notes: not served here") instead of failing.

+

The other way is a registry: an extension point one plugin defines and others contribute to. The app's API +defines Panels in its web module, and every plugin with a UI contributes one:

+
// plugins/notes/web.tsx
+import { Panels } from "@app/api/v1/web";
+export default defineFacet((host, self) => self.contribute(Panels, { title: "Notes", Component: NotesPanel }));
+

What a plugin may not do is reach into another's data. Search never reads notes' documents; it uses the Notes +object's state. That rule is what lets the notes plugin change its storage without breaking search.

+
+ +
VIII

Changing a Plugin

Edit, build, roll out: how new code takes over while every tab stays connected.

+

You change the notes plugin: notes now have a timestamp. The watcher sees the saved file and marks the plugin dirty. +You (or an agent) build and roll out:

+
const built = await plugins.build(plugins.state.value.dirty, ctx);
+const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx);
+

The rollout goes host by host, server first, then every running worker. On each host Chord compares the new plugin +set with the loaded one, runs the new factories against a recording self to see what each object would declare, and +then gives every object of the changed plugin its fate:

+ + + + + + + + + + + + + + + + + + + +
The objectWhat happens
holds no natives (open files, sockets), the new code neitherswapped in place: new methods, same documents; running calls finish on the old code
holds nativesrestarted: calls wait, running ones finish, natives close and reopen with the new code
its type was removedended; its users wait until something serves it again
+

The tab's reference follows the new object and gets one frame with the difference. Nothing reconnects. If the edit +is broken (a syntax error, a factory that throws), that plugin keeps its old code on that host and the report says why; +every other plugin of the rollout applies.

+
Versions

A change to a contract (a new field in Notes's state) gets a new version. A tab built with the old contract keeps +working if the provider maps the old version (accepts); otherwise it waits with "version mismatch" until it reloads.

+
+
+ +
IX

Base Code and Plugins

What goes into the app itself, what goes into plugins, and how an app ships plugins of its own.

+

Every app has two kinds of code, and the line between them decides what can change while the app runs.

+

Base code is the app's own program: src/, started by node server.ts. It shrinks to three things: the API +(the versioned modules plugins import), the bootstrap (creating hosts, opening the store, components such as +createHttp and createWorkers) and one small start-data facet serving process values (start data, the opened +store) as core objects. Base code changes only with a restart: if it reloaded, the process would hold two Panels +registries, and old and new plugins would no longer meet.

+

Plugins are everything else: notes, search, export, a terminal, the harness, the providers of the app's core +registries. They reload on an edit, a broken one is left out alone, and an agent can write one while the app runs.

+

The API, by version and by kind

+

Plugins import their host only through the API, a version in the path:

+
// src/api/v1/index.ts: "@app/api/v1", for every kind
+export { defineRegistry, local, obj, remote, str } from "@earendil-works/chord/next";   // the Chord surface the app supports
+export { defineFacet, Project, Server } from "../../core.ts";                           // bound to the app's principal
+// src/api/v1/web.ts: "@app/api/v1/web", for pages only
+export const Panels = defineRegistry<Panel, readonly Panel[]>("app.panels", { version: 1 });
+

The app lists the versions it provides, newest first (api: { name: "@app/api", versions: ["v2", "v1"] }), and each +kind's modules in kinds.<kind>.hostPackages. When the app changes its API in a way that breaks plugins, it adds +/v2 and keeps /v1 as a thin layer over it for a while; once /v1 goes, a plugin importing it fails to build with +"@app/api/v1 is not available: this app provides v2", the plugin alone, the message saying what to change. One facet +uses one version. Contracts the app promises others are re-exported in the API (export { Notes } from "../../plugins/notes/contract.ts"), so a plugin built against /v1 keeps v1's contract files.

+

Plugins the app ships

+

An app comes with plugins of its own. They follow the same rules as anyone's, in a root inside the app, next to the +root where users and agents add theirs:

+
const plugins = createPlugins({
+	roots: [{ path: "./plugins", required: true }, join(dataDir, "plugins")],   // the app's own, then the user's
+	api: { name: "@app/api", versions: ["v1"] },
+	kinds,
+	onError,
+}, ctx);
+

They are built at startup like any plugin (a quarter of a second for twenty), reload like any plugin, and are left out +alone when broken at a reload. The app's root is required: a plugin of it failing at startup makes +plugins.load reject with a PluginError naming it (after the load settled), so the app stops instead of running +without its core. A failure in the user's root is reported and the app runs on. If two roots have a plugin with the +same name, the first root wins, and the other is reported.

+

Small plugins, installed plugins, tested plugins

+
    +
  • A single file at a root's top is a plugin: plugins/hello.server.ts and plugins/hello.web.tsx are the plugin +hello with two facets. It has no package.json and imports only the API; when it needs a second file or a +package, it becomes the directory hello/, which a reload treats as the same plugin.
    • +
  • Installing from git or npm: plugins.installPlugin({ git: url, ref }, root, ctx) clones the ref (a branch, tag +or commit) into the root, { npm: name, version } extracts the package; then its dependencies are installed. Removal +is deleting the directory.
    • +
  • Testing a plugin alone: testPlugin("./plugins/stats", { kind: "worker", api, modules, stubs }, ctx) builds it +as production does, loads it on an in-memory host beside stub facets serving what it uses, collects what it +contributes to registries nobody serves, and lists the contracts it waits for (docs/next/plugin-authoring.md).
    • +
+
+ +
X

The App Kit

startServer and startWorker: the hosts, stores, sign-in, files and plugin roots every app needs, so an app is its API, its plugins and a start script.

+

The chapters so far wired an app by hand: a store, a host, createHttp, createWorkers, createPlugins, an +authenticator, a worker entry connecting to its parent. Every app needs the same wiring, and most of it has one right +answer. The app kit (chord/next/app) is that answer, opinionated on purpose: one data directory layout, one way +to sign in, one place for files.

+

A whole app

+

The notes app of example 53 is four files of base code and two plugins:

+
// start.ts
+const app = await startServer<NotesPrincipal>({
+	name: "notes",                                  // the data directory: ~/.notes unless dataDir says otherwise
+	root: "./plugins",                              // the app's own plugins: a required root
+	api: { name: "@notes/app", versions: ["v1"], modules: { "@notes/app/v1": v1, "@notes/app/v1/server": v1server } },
+	kinds: { server: { platform: "node" }, worker: { platform: "node" } },
+	principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", project: address.key }) },
+	workers: [{
+		kind: Project,                              // one worker per project
+		entry: "./worker.ts",
+		start: (address) => ({ project: address.key }),
+		authorize: (principal) => principal.kind === "owner" || principal.kind === "server",
+	}],
+	onError: (error) => log(error),
+}, ctx);
+console.log(`Open ${app.ownerLink()}`);
+
+// worker.ts
+startWorker({ api: API, kinds: KINDS, onError: (error) => log(error) });
+

startServer creates the data directory (server/store.sqlite, workers/<kind>/<address>/, plugins/, logs/, +owner-token), opens the server's store, creates the server host, HTTP, a workers component per worker kind and the +plugins component over two roots: the app's plugins/ and the user's <dataDir>/plugins/. Then it loads the server's +plugins with the kit's own facets beside them and resolves. startWorker does the worker's half: the project's +directory and store, the host at the project's address, the link to the server first, the worker's plugins.

+

What plugins get from the kit

+

Plugins still import only the app's API; the API re-exports the kit's objects it offers:

+
    +
  • AppInfo on the server (data directory, app name, API version) and WorkerInfo on workers (the address and the +start data start(address) made): the start-data objects. Process values reach plugins only through them.
    • +
  • Files, one per host: put for small files, begin, append and finish for large ones, ranged read, remove +and a watchable list, stored in the host's directory with caps per host.
    • +
  • Sessions on the server: issue, revoke, list.
    • +
+

This finishes what the previous chapter started. Base code is the API, the start script and the worker's entry. The +providers of the app's own registries live in its required root: example 53's API declares a Sections registry, and +its home plugin serves it and reads AppInfo, so it reloads like any other plugin.

+

Who may come in

+

HTTP denies by default: a request without a principal gets 401 before any plugin code runs, unless the route says +public: true. Someone gets a principal through a session: a random token, kept in the store only as a hash, sent +as a signed cookie by a browser or as Authorization: Bearer by a command-line client.

+

The first session is the owner's. At each start the kit makes a one-time owner link, +http://127.0.0.1:8788/_chord/owner#<token>, valid ten minutes; the start script prints it. The token is in the +fragment, which browsers never send, so it never reaches a log: the page posts it, the kit consumes it and sets the +owner's cookie. A command-line client on the same machine reads <dataDir>/owner-token (mode 0600). One on another +machine pairs: it asks for a code, the owner approves the code in a signed-in browser, and the client's next poll +returns its token (example 54).

+

Users other than the owner come from optional plugins the app adds: defineUsers(profile) for accounts, +inviteFacet() for invite links, ephemeralLoginFacet() for "pick a name and you're in".

+

Applying changes, moving projects

+

The kit never reloads by itself. An edit or an install marks plugins dirty; app.plugins.state drives the app's +"apply changes" button, and app.applyChanges(ctx) builds and rolls out to the server, then the workers.

+

A project's data is what its worker created: its store and its files. app.exportData(Project({ id }), path, ctx) +writes them as one archive (a manifest, the store's export, the files) and stops nothing: a running worker writes its +own store at one consistent point. app.importData(path, Project({ id: "copy" }), ctx) makes a new project from it and +refuses one that already has data (example 55).

+
+ +
A

Reference

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CallWhat it does
createHost<P>({ address?, principal?, store?, onError })a host: addressable with an address, else a client
host.load([{ name, facet }], ctx) / host.reload(...)publish facets / apply a new plugin set
host.get(Type, params, ctx) / host.hold(...)a live reference / a reference at once that follows its object
local(name, { version, params, members }) / remote(name, { version, params, hostAddress, variants })contracts
defineFacet((host, self) => …), host.serve(Type, factory), self.contribute(R, value)a facet
factory self: state, derive, get, registry, open, on, onLive, job, txan object's parts
createPlugins({ roots, api, kinds, onError }, ctx)the plugins component
plugins.load(host, { kind, core, modules }, ctx), build(names), rollout(steps, built)load, build, roll out
plugins.installPlugin(source, root, ctx), testPlugin(dir, options, ctx)install a plugin, test one alone
hostKind(name, params?), Kind(params)host kinds and addresses
startServer<P>({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, onError }, ctx)an app's server, with the kit
startWorker({ api, kinds, busy?, onError })a worker's entry, with the kit
app.ownerLink(), app.applyChanges(ctx), app.plugins.install(source, ctx)sign in, apply changes, install
app.exportData(address, path, ctx), app.importData(path, address, ctx)move a worker's data
AppInfo, WorkerInfo, Files, Sessionsthe kit's objects plugins get
+
+ +

Hosts and Plugins · third edition · 2026-11-02 · Chord v2 slices 1-10, 12 and 13

+
+ + diff --git a/packages/chord/docs/next/http-workers/book.md b/packages/chord/docs/next/http-workers/book.md new file mode 100644 index 000000000..f340004ef --- /dev/null +++ b/packages/chord/docs/next/http-workers/book.md @@ -0,0 +1,378 @@ +--- +title: Serving and Spawning: HTTP and workers in Chord v2 +cover: Serving / and Spawning +kicker: An explainer · Chord v2 slice 10 · 2026-10-31 +subtitle: How a request reaches a plugin's route, who it acts as, and how a worker process per session starts, stops and fails +stats: + - 1 | registry plugins contribute routes, wraps, sockets and hosts to + - 0 | automatic worker restarts + - 5 min | idle before a worker stops (default) + - 6 | security rules on by default +note: | + Written by the redesign session from `slice-10.md`, the decision entries "Slice 10's points", "Origins only from + relays", "The HTTP component's security baseline" and their neighbours in `decisions.md`, design §3 and §23, and the + build after its four fix rounds (security baseline, `fetch` core, error boundaries). +colophon: Serving and Spawning · first edition · 2026-10-31 · Chord v2 slice 10 +--- + +# Before we begin {.preface} + +Mario runs a small chat app on his laptop. Each conversation is a **session**, and each session runs in its own +process, a **worker**, so a session that crashes or eats memory takes nothing else down. A browser tab talks to the +server; the server talks to the workers. Plugins add features: one adds a chat panel, one adds file downloads, one +adds a terminal. Mario edits plugins while the app runs, and sometimes an agent edits them for him. + +Three things have to work for him without thought. A plugin adds an HTTP route, and it is served safely: no other +website open in his browser can read his data or act as him through it. A session's worker exists exactly while +someone uses it. And when a worker crashes, nothing restarts it in a loop: he sees why it failed and decides. + +Slice 10 builds two components for this: `chord/next/http` (the server's HTTP and WebSocket entry) and +`chord/next/workers` (a worker process per session). Both are Node code on top of Chord's kernel; the kernel itself +stays portable. + +## Eight things to know + +:::findings +1. **Plugins contribute routes, Chord serves them.** The HTTP component defines one registry, `Http`; plugins + contribute routes, wraps (middleware), WebSocket handlers and allowed hosts to it, and every reload swaps them. +2. **A route returns a description.** JSON, HTML, bytes, a redirect or an error; the component writes it, with ranges, + `HEAD`, download headers and backpressure done once for every plugin. +3. **Security is built in.** Where a request came from, whether another website sent it, and which headers every + response carries are decided by the component before any plugin code runs. The app only says who someone is. +4. **A worker starts on the first use of its session** and stops after five idle minutes. The server forks it; the + worker connects back and loads its plugins. +5. **A worker never restarts by itself.** A crash or a failed start leaves the session failed with its error until a + person presses Retry or rolls out new code. +6. **Workers never build plugins.** The server builds every plugin once; a worker evaluates the scripts it is sent. +7. **A rollout reaches every host.** The server reloads first, then every running worker, each reporting its result. +8. **Everything but the bootstrap is changeable by plugins.** Routes, hosts, start data, idle times: contributions or + functions read at each use. +::: + +## Words used throughout + +A **route** is a function a plugin contributes for a method and path pattern. A **wrap** runs around routes. A +**socket** is a WebSocket a plugin handles with its own protocol; a **link** is Chord's own WebSocket at `/chord`, +over which tabs hold objects. **Admission** is what the component checks before any plugin code. **Provenance** is +where a request came from (local, an allowed host, a relay). A request is **foreign** when a browser sent it for +another website. A **worker** is a process serving one **address** (`Session({ id })`). A worker's **start message** +carries its start data and its plugins' **scripts**. A **rollout** applies a build to the server and then the workers. + +# The App We Build {#app lede="A server, workers, two plugins: the files a reader needs to follow the rest."} + +The whole app is four files of its own plus its plugins. The server creates its host, the HTTP component and the +workers component, then loads its plugins: + +```ts +// server.ts +const server = createHost

({ address: Server(), principal: { kind: "server" }, onError }); +const plugins = createPlugins({ + roots: ["./plugins"], + kinds: { + server: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] }, + worker: { platform: "node", hostPackages: ["@earendil-works/chord/next", "@app/core"] }, + }, + onError, +}, ctx); +const http = createHttp(server, { listen: { host: "127.0.0.1", port: 8788 }, onError }, ctx); // settings are arguments +const workers = createWorkers(server, Session, { + kind: "worker", entry: "./worker.ts", plugins, principal: (a) => ({ kind: "worker", session: a.params.id }), + start: (address, ctx) => ({ id: address.params.id, dir: sessionDir(address) }), + lock: (address) => `chat:${dataDir}:${address.key}`, authorize: mayReach, onError, +}, ctx); +await plugins.load(server, { kind: "server", core: [core, { name: "chord.http", facet: http.facet }], modules }, ctx); +// `core` contributes the app's authenticator: self.contribute(Authenticators, authenticate) +``` + +```ts +// worker.ts: one per session +await workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(start.dir, "session.sqlite"))), ctx); + onClose((ctx) => store.close(ctx)); + const worker = createHost

({ address, principal: { kind: "worker", session: start.id }, store, onError }); + onClose((ctx) => worker.close(ctx)); + worker.connect(parent, { principal: { kind: "server" }, trusted: true }, ctx); + await plugins.load(worker, { kind: "worker", core: [workerCore(start)] }, ctx); + return { busy: { tasks: 0 } }; +}, { hostPackages: modules, redact: (error) => ({ message: "worker failed" }), onError }); +``` + +A plugin is a directory with a `package.json` and one file per kind it needs. The chat plugin keeps its messages in a +worker (each session has its own) and contributes one route on the server for exporting them: + +```ts +// plugins/chat/server.ts +export default defineFacet((host, self) => { + self.contribute(Http, { + routes: { + "GET /s/:session/export": async (request, ctx) => { + if (!(await mayView(ctx.principal, request.params.session, ctx))) return { kind: "error", status: 403 }; + const chat = await ctx.get(Chat, { session: request.params.session }); // reaches the session's worker + return { kind: "json", value: await chat.export(ctx) }; + }, + }, + }); +}); +``` + +`ctx.get(Chat, { session })` is the one line that connects the two halves: `Chat`'s host address is the session's +worker, the server's route for `Session` finds (or starts) that worker, and the call goes over its link. + +# A Request, Start to End {#request lede="What happens between a TCP connection and the route's response, in order."} + +Every request takes the same path. The order matters: nothing a plugin wrote runs before the component decided that +the request may run at all, and as whom. + +:::flow wide animate | One request through the HTTP component. Everything left of `authenticate` is the component's own; plugins start at the router. +row "Admission" accent +box prov "Provenance" "local, allowed host or relay; else 403" +box foreign "Foreign?" "another website's browser request" +box origin "Origin" "foreign Origin 403; unsafe methods need ours" +row "Who" blue +box auth "authenticate" "the app: cookies, devices" +box anon "anonymous" "foreign or no credential" muted +row "Plugins" +box router "Router" "pattern → route, as of now" +box wraps "Wraps" "observe, headers, transform" +box route "Route" "returns a description" +row "Answer" muted +box write "Write" "headers, ranges, chunks, backpressure" +arrow prov -> foreign +arrow foreign -> origin +arrow origin -> auth +arrow origin -> anon "foreign" +arrow auth -> router +arrow router -> wraps +arrow wraps -> route +arrow route -> write +::: + +**Provenance** answers "where from". A request is local only if three things hold: the socket arrived on a loopback +address, the `Host` header names loopback with the port, and no proxy header (`Forwarded`, `X-Forwarded-*`) is present. +Anything else must name a host the app allowed (a tunnel's public name) and is then treated as remote https. Anything +else is refused before a single plugin line runs. The security audit found that checking the `Host` header alone let +anyone reaching the port claim to be local: the socket address is the part a remote client cannot fake. + +**Foreign** answers "did another website make the browser send this". Browsers say so in `Sec-Fetch-Site`. A foreign +request never reaches `authenticate`: it runs anonymous, whatever cookies it carries. That one rule stops a page on +another site from reading Mario's data through an `` or ` + diff --git a/packages/chord/docs/next/i18n/book.md b/packages/chord/docs/next/i18n/book.md new file mode 100644 index 000000000..b431cb030 --- /dev/null +++ b/packages/chord/docs/next/i18n/book.md @@ -0,0 +1,216 @@ +--- +title: Speaking Every Language: i18n in the app kit +cover: i18n / in the App Kit +kicker: A proposal · Chord v2 app kit · 2026-11-02 +subtitle: How an app and its plugins write their texts once, in code, and show them in each viewer's language +stats: + - 1 | function a plugin calls: t(key, ...args) + - 0 | message files outside the code + - 0 | page reloads to switch a language +note: | + A proposal by the redesign session, asked for by the owner on 2026-11-02 ("i18n also needs to be part of the app + kit"; "need code example"). It starts from mini-lit's `src/i18n.ts` (typed catalogs in code) and adapts it to + plugins that are built separately, reload live and run on several hosts. Not built; it would land in the app kit + with slice 11. +colophon: Speaking Every Language · proposal · 2026-11-02 +--- + +# Before we begin {.preface} + +Lena uses p3 in German; her colleague Tom, a guest in her session, reads English. The notes plugin shows "New note" +to Tom and "Neue Notiz" to Lena, on the same page served by the same server. A French translator later adds French to +the app and to every plugin, without touching any of them. And when Lena switches to French, nothing reloads: the +text on her screen changes, the half-written message stays. + +mini-lit already does the first part well: messages are TypeScript objects, keyed and type-checked, so a missing +German text is a compile error. Three things do not carry over: its messages are one global table extended by +declaration merging (which separately built plugins cannot share), the language lives in `localStorage` and switching +reloads the page, and every text is formatted where it is shown, which breaks when the server formats a message for +two viewers with two languages. + +## Five things to know + +:::findings +1. **Messages stay in code,** per plugin: `defineMessages("notes", { en: {...}, de: {...} })` returns a typed `t`. +2. **Values may be functions,** for plurals and parameters; dates and numbers go through `Intl`. +3. **Languages are contributions:** a `Translations` registry adds a language to any plugin, the app included. +4. **The language is a user preference,** from the browser or the system locale by default; switching re-renders live. +5. **Servers and workers never format text for people:** they send a key and its parameters, and each viewer's client + renders them in its own language. +::: + +## Words used throughout + +A **message** is one text with a key (`newNote`). A **catalog** is the messages of one plugin in one language. A +**namespace** is the plugin a catalog belongs to. A **locale** is a language tag (`de`, `pt-BR`). + +# Messages in Code {#messages lede="Each plugin defines its own messages and gets a typed function back."} + +```ts +// plugins/notes/messages.ts +import { defineMessages } from "@p3/app/v1"; + +export const t = defineMessages("notes", { + en: { + newNote: "New note", + empty: "No notes yet", + count: (n: number) => (n === 1 ? "1 note" : `${n} notes`), + saved: (title: string, at: Date) => `Saved "${title}" at ${formatTime(at)}`, + }, + de: { + newNote: "Neue Notiz", + empty: "Noch keine Notizen", + count: (n) => (n === 1 ? "1 Notiz" : `${n} Notizen`), + saved: (title, at) => `„${title}“ um ${formatTime(at)} gespeichert`, + }, +}); +``` + +```tsx +// plugins/notes/web.tsx +import { t } from "./messages.ts"; + +function NotesHeader({ count }: { count: number }) { + return

{t("count", count)}

; +} +``` + +The first catalog (`en`) defines the keys and their argument types; every other catalog must have the same keys with +the same signatures, so a missing German text, a typo in a key or a wrong argument is a type error, not a blank in +production. `t("count", "3")` does not compile. The namespace (`"notes"`) keeps two plugins' `empty` apart. + +Functions cover what plain strings cannot: plurals (`count`), parameters in the right word order (`saved` puts the +title first in English and German alike, but another language may not), and gender or case where a language needs it. +Plural rules beyond one and many use `Intl.PluralRules`; dates, times and numbers use `Intl.DateTimeFormat` and +`Intl.NumberFormat` with the viewer's locale, through two small helpers (`formatTime`, `formatNumber`) that read the +current locale like `t`. + +# Languages as Contributions {#translations lede="Adding French to the app and every plugin without editing them."} + +A plugin ships the languages its author speaks. Everything else can come from anyone, as a contribution: + +```ts +// plugins/french/web.ts (a translation plugin) +import { defineFacet, Translations } from "@p3/app/v1"; +import type { MessagesOf } from "@p3/app/v1"; +import type { t as notes } from "../notes/messages.ts"; // the notes plugin's keys, for the type check + +const notesFr: MessagesOf = { + newNote: "Nouvelle note", + empty: "Aucune note", + count: (n) => (n <= 1 ? `${n} note` : `${n} notes`), + saved: (title, at) => `« ${title} » enregistrée à ${formatTime(at)}`, +}; + +export default defineFacet((_host, self) => { + self.contribute(Translations, { namespace: "notes", locale: "fr", messages: notesFr }); + self.contribute(Translations, { namespace: "app", locale: "fr", messages: appFr }); +}); +``` + +- `MessagesOf` gives a translation plugin the exact keys and signatures to fill, from the plugin it + translates (an imported type, erased at build). +- A catalog may be partial: a missing key falls back to the plugin's own catalog for that language's base (`pt-BR` to + `pt`), then to the first catalog (`en`), key by key. A fallback is reported once in the logs, so gaps are visible. +- Two contributions for the same namespace and locale: the first by plugin name wins, the other is reported (the + registry rule for conflicts). +- A plugin's own catalogs are its built-in contributions; a translation plugin can also override one of them (a + better German), contributed explicitly with `override: true`. + +## One file, every host + +Registries are per host: a contribution lives on the host whose facet made it. A French catalog contributed from the +web facet exists in browser pages only. So a plugin keeps its messages in one shared file, `messages.ts`, which every +facet that renders text imports (the web and TUI facets; the worker's too when it builds messages with `t.message`), +and a translation plugin keeps its catalog in one shared file too and contributes it from each facet that renders: + +```ts +// plugins/french/fr.ts: the catalogs, written once +export const notesFr: MessagesOf = { newNote: "Nouvelle note", /* ... */ }; +// plugins/french/web.ts and plugins/french/tui.ts: the same two lines each +export default defineFacet((_host, self) => self.contribute(Translations, { namespace: "notes", locale: "fr", messages: notesFr })); +``` + +The server and workers need catalogs only when they produce final text themselves (an email, a push notification); +then their facets contribute them too, and the text is rendered in the recipient's language from their preference. +Catalogs never travel over links: they contain functions. + +# Choosing a Language {#locale lede="Per viewer, defaulted from the device, switched live."} + +The language is a user preference, kept with the user's other preferences (the kit's per-principal document, like +the theme choice), so it follows a person across devices: + +- **Default:** the browser's `navigator.languages` on the web, the system locale in the TUI, matched against the + locales any catalog offers. +- **Switching:** the app's settings show the locales available (the union of all catalogs); choosing one writes the + preference. `t` reads the current locale from a watchable; React components re-render through a hook, the TUI + redraws through its scope. Nothing reloads. + +```tsx +// any web component +const { t, locale } = useMessages(notesMessages); // re-renders when the locale or a translation changes +``` + +When a translation plugin is edited and rolled out, the registry recomposes and the open pages show the new text, as +with themes. + +# Text from the Server {#server lede="Why the server sends keys, not sentences."} + +The server and the workers also produce text people read: a notice ("Worker failed to start"), a tool card's +summary, a status line, an error. If the worker formatted "Notiz gespeichert" for Lena, Tom would read German too: +both look at the same object state. + +So objects carry messages as data, and each client renders them: + +```ts +// plugins/notes/contract.ts: a member of the Notes object's state holds a message as data +lastSaved: opt(obj({ namespace: str(), key: str(), params: arr(unknown()) })), + +// plugins/notes/worker.ts: the object writes the key and its parameters, never a sentence; the same `t` builds it +import { t } from "./messages.ts"; +await self.tx(async (tx) => { + (await tx.draft(status)).lastSaved = t.message("saved", note.title, note.at); // typed: key and arguments checked +}, ctx); + +// plugins/notes/web.tsx: each viewer renders it in its own language +const notes = useHold(Notes, { session }); +const text = notes.lastSaved && t.render(notes.lastSaved); // typed against the notes catalog +``` + +`t.message` returns a `Message<"notes">`, plain data `{ namespace, key, params }`, typed against the first catalog: +`t.message("saved", 42)` and `t.message("svaed", ...)` are type errors. `t.render` takes only `Message<"notes">`. Across a +link the contract's member is a generic message schema, so `t.render` checks the namespace and key at runtime and +shows the key for an unknown one. The writer and the reader are type-checked; the hop between them is checked at +runtime. + +The same holds for anything else that reaches people: an app's notices object stores `{ namespace, key, params }` +per notice, a tool card's summary carries one, an error shown to a user carries a code and parameters. + +Parameters are plain data (strings, numbers, ISO dates), so they cross links like any state. What is never +translated: the model's prompts and the model's own answers (the conversation's language is the user's business with +the model), log lines (for developers, in English), error codes. Error *messages* shown to people follow the same rule +as notices: a code plus parameters, rendered by the client. + +# Open Questions {.epilogue lede="To decide."} + +1. **Keys or English text as keys.** mini-lit keys by the English text (`"Copy"`), which reads well and needs no key + names; named keys (`newNote`) survive rewording the English. Recommendation: named keys, with the English catalog + as the source of truth. +2. **A file format for translators.** Code catalogs are best for developers and agents; human translators may want + JSON or PO files. Recommendation: no files now; a small tool can export a plugin's catalogs to JSON and import them + back into a translation plugin when someone needs it. +3. **Right-to-left languages.** The page sets `dir` from the locale; the app's CSS uses logical properties + (`margin-inline-start`). Recommendation: the kit sets `lang` and `dir` on the root; components are the app's. +4. **The TUI's locale.** From the system locale, overridden by the user's preference, like the web. + +# API in Brief {.appendix} + +| Piece | Where | What | +| --- | --- | --- | +| `defineMessages(namespace, catalogs)` | the app's API (shared) | typed `t(key, ...args)`, `t.render(message)` | +| `MessagesOf` | the app's API (type) | the keys and signatures a translation must fill | +| `Translations` | the app's API (shared registry) | `{ namespace, locale, messages, override? }` | +| `useMessages(t)` | the app's web module | the hook re-rendering on locale or translation changes | +| `formatTime`, `formatNumber` | the app's API | `Intl` helpers in the viewer's locale | +| the locale preference | the kit's per-principal preferences | default from the device, switched live | +| `{ namespace, key, params }` | object state and notices | a message as data, rendered by each client | diff --git a/packages/chord/docs/next/i18n/index.html b/packages/chord/docs/next/i18n/index.html new file mode 100644 index 000000000..7a2fb4888 --- /dev/null +++ b/packages/chord/docs/next/i18n/index.html @@ -0,0 +1,343 @@ + + +Speaking Every Language: i18n in the app kit + + + + +
+
+
A proposal · Chord v2 app kit · 2026-11-02
+

i18n
in the App Kit

+

How an app and its plugins write their texts once, in code, and show them in each viewer's language

+
1function a plugin calls: t(key, ...args)
0message files outside the code
0page reloads to switch a language
+

A proposal by the redesign session, asked for by the owner on 2026-11-02 ("i18n also needs to be part of the app +kit"; "need code example"). It starts from mini-lit's src/i18n.ts (typed catalogs in code) and adapts it to +plugins that are built separately, reload live and run on several hosts. Not built; it would land in the app kit +with slice 11.

+
+
+ + +
+

Before we begin

+

Lena uses p3 in German; her colleague Tom, a guest in her session, reads English. The notes plugin shows "New note" +to Tom and "Neue Notiz" to Lena, on the same page served by the same server. A French translator later adds French to +the app and to every plugin, without touching any of them. And when Lena switches to French, nothing reloads: the +text on her screen changes, the half-written message stays.

+

mini-lit already does the first part well: messages are TypeScript objects, keyed and type-checked, so a missing +German text is a compile error. Three things do not carry over: its messages are one global table extended by +declaration merging (which separately built plugins cannot share), the language lives in localStorage and switching +reloads the page, and every text is formatted where it is shown, which breaks when the server formats a message for +two viewers with two languages.

+

Five things to know

+
    +
  1. Messages stay in code, per plugin: defineMessages("notes", { en: {...}, de: {...} }) returns a typed t.
    2. +
  2. Values may be functions, for plurals and parameters; dates and numbers go through Intl.
    3. +
  3. Languages are contributions: a Translations registry adds a language to any plugin, the app included.
    4. +
  4. The language is a user preference, from the browser or the system locale by default; switching re-renders live.
    5. +
  5. Servers and workers never format text for people: they send a key and its parameters, and each viewer's client +renders them in its own language.
    6. +
+

Words used throughout

+

A message is one text with a key (newNote). A catalog is the messages of one plugin in one language. A +namespace is the plugin a catalog belongs to. A locale is a language tag (de, pt-BR).

+
+ +
I

Messages in Code

Each plugin defines its own messages and gets a typed function back.

+
// plugins/notes/messages.ts
+import { defineMessages } from "@p3/app/v1";
+
+export const t = defineMessages("notes", {
+	en: {
+		newNote: "New note",
+		empty: "No notes yet",
+		count: (n: number) => (n === 1 ? "1 note" : `${n} notes`),
+		saved: (title: string, at: Date) => `Saved "${title}" at ${formatTime(at)}`,
+	},
+	de: {
+		newNote: "Neue Notiz",
+		empty: "Noch keine Notizen",
+		count: (n) => (n === 1 ? "1 Notiz" : `${n} Notizen`),
+		saved: (title, at) => `„${title}“ um ${formatTime(at)} gespeichert`,
+	},
+});
+
// plugins/notes/web.tsx
+import { t } from "./messages.ts";
+
+function NotesHeader({ count }: { count: number }) {
+	return <h2>{t("count", count)} <button>{t("newNote")}</button></h2>;
+}
+

The first catalog (en) defines the keys and their argument types; every other catalog must have the same keys with +the same signatures, so a missing German text, a typo in a key or a wrong argument is a type error, not a blank in +production. t("count", "3") does not compile. The namespace ("notes") keeps two plugins' empty apart.

+

Functions cover what plain strings cannot: plurals (count), parameters in the right word order (saved puts the +title first in English and German alike, but another language may not), and gender or case where a language needs it. +Plural rules beyond one and many use Intl.PluralRules; dates, times and numbers use Intl.DateTimeFormat and +Intl.NumberFormat with the viewer's locale, through two small helpers (formatTime, formatNumber) that read the +current locale like t.

+
+ +
II

Languages as Contributions

Adding French to the app and every plugin without editing them.

+

A plugin ships the languages its author speaks. Everything else can come from anyone, as a contribution:

+
// plugins/french/web.ts (a translation plugin)
+import { defineFacet, Translations } from "@p3/app/v1";
+import type { MessagesOf } from "@p3/app/v1";
+import type { t as notes } from "../notes/messages.ts";   // the notes plugin's keys, for the type check
+
+const notesFr: MessagesOf<typeof notes> = {
+	newNote: "Nouvelle note",
+	empty: "Aucune note",
+	count: (n) => (n <= 1 ? `${n} note` : `${n} notes`),
+	saved: (title, at) => `« ${title} » enregistrée à ${formatTime(at)}`,
+};
+
+export default defineFacet((_host, self) => {
+	self.contribute(Translations, { namespace: "notes", locale: "fr", messages: notesFr });
+	self.contribute(Translations, { namespace: "app", locale: "fr", messages: appFr });
+});
+
    +
  • MessagesOf<typeof t> gives a translation plugin the exact keys and signatures to fill, from the plugin it +translates (an imported type, erased at build).
    • +
  • A catalog may be partial: a missing key falls back to the plugin's own catalog for that language's base (pt-BR to +pt), then to the first catalog (en), key by key. A fallback is reported once in the logs, so gaps are visible.
    • +
  • Two contributions for the same namespace and locale: the first by plugin name wins, the other is reported (the +registry rule for conflicts).
    • +
  • A plugin's own catalogs are its built-in contributions; a translation plugin can also override one of them (a +better German), contributed explicitly with override: true.
    • +
+

One file, every host

+

Registries are per host: a contribution lives on the host whose facet made it. A French catalog contributed from the +web facet exists in browser pages only. So a plugin keeps its messages in one shared file, messages.ts, which every +facet that renders text imports (the web and TUI facets; the worker's too when it builds messages with t.message), +and a translation plugin keeps its catalog in one shared file too and contributes it from each facet that renders:

+
// plugins/french/fr.ts: the catalogs, written once
+export const notesFr: MessagesOf<typeof notes> = { newNote: "Nouvelle note", /* ... */ };
+// plugins/french/web.ts and plugins/french/tui.ts: the same two lines each
+export default defineFacet((_host, self) => self.contribute(Translations, { namespace: "notes", locale: "fr", messages: notesFr }));
+

The server and workers need catalogs only when they produce final text themselves (an email, a push notification); +then their facets contribute them too, and the text is rendered in the recipient's language from their preference. +Catalogs never travel over links: they contain functions.

+
+ +
III

Choosing a Language

Per viewer, defaulted from the device, switched live.

+

The language is a user preference, kept with the user's other preferences (the kit's per-principal document, like +the theme choice), so it follows a person across devices:

+
    +
  • Default: the browser's navigator.languages on the web, the system locale in the TUI, matched against the +locales any catalog offers.
    • +
  • Switching: the app's settings show the locales available (the union of all catalogs); choosing one writes the +preference. t reads the current locale from a watchable; React components re-render through a hook, the TUI +redraws through its scope. Nothing reloads.
    • +
+
// any web component
+const { t, locale } = useMessages(notesMessages);   // re-renders when the locale or a translation changes
+

When a translation plugin is edited and rolled out, the registry recomposes and the open pages show the new text, as +with themes.

+
+ +
IV

Text from the Server

Why the server sends keys, not sentences.

+

The server and the workers also produce text people read: a notice ("Worker failed to start"), a tool card's +summary, a status line, an error. If the worker formatted "Notiz gespeichert" for Lena, Tom would read German too: +both look at the same object state.

+

So objects carry messages as data, and each client renders them:

+
// plugins/notes/contract.ts: a member of the Notes object's state holds a message as data
+lastSaved: opt(obj({ namespace: str(), key: str(), params: arr(unknown()) })),
+
+// plugins/notes/worker.ts: the object writes the key and its parameters, never a sentence; the same `t` builds it
+import { t } from "./messages.ts";
+await self.tx(async (tx) => {
+	(await tx.draft(status)).lastSaved = t.message("saved", note.title, note.at);   // typed: key and arguments checked
+}, ctx);
+
+// plugins/notes/web.tsx: each viewer renders it in its own language
+const notes = useHold(Notes, { session });
+const text = notes.lastSaved && t.render(notes.lastSaved);   // typed against the notes catalog
+

t.message returns a Message<"notes">, plain data { namespace, key, params }, typed against the first catalog: +t.message("saved", 42) and t.message("svaed", ...) are type errors. t.render takes only Message<"notes">. Across a +link the contract's member is a generic message schema, so t.render checks the namespace and key at runtime and +shows the key for an unknown one. The writer and the reader are type-checked; the hop between them is checked at +runtime.

+

The same holds for anything else that reaches people: an app's notices object stores { namespace, key, params } +per notice, a tool card's summary carries one, an error shown to a user carries a code and parameters.

+

Parameters are plain data (strings, numbers, ISO dates), so they cross links like any state. What is never +translated: the model's prompts and the model's own answers (the conversation's language is the user's business with +the model), log lines (for developers, in English), error codes. Error messages shown to people follow the same rule +as notices: a code plus parameters, rendered by the client.

+
+ +
∴

Open Questions

To decide.

+
    +
  1. Keys or English text as keys. mini-lit keys by the English text ("Copy"), which reads well and needs no key +names; named keys (newNote) survive rewording the English. Recommendation: named keys, with the English catalog +as the source of truth.
    2. +
  2. A file format for translators. Code catalogs are best for developers and agents; human translators may want +JSON or PO files. Recommendation: no files now; a small tool can export a plugin's catalogs to JSON and import them +back into a translation plugin when someone needs it.
    3. +
  3. Right-to-left languages. The page sets dir from the locale; the app's CSS uses logical properties +(margin-inline-start). Recommendation: the kit sets lang and dir on the root; components are the app's.
    4. +
  4. The TUI's locale. From the system locale, overridden by the user's preference, like the web.
    5. +
+
+ +
A

API in Brief

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PieceWhereWhat
defineMessages(namespace, catalogs)the app's API (shared)typed t(key, ...args), t.render(message)
MessagesOf<typeof t>the app's API (type)the keys and signatures a translation must fill
Translationsthe app's API (shared registry){ namespace, locale, messages, override? }
useMessages(t)the app's web modulethe hook re-rendering on locale or translation changes
formatTime, formatNumberthe app's APIIntl helpers in the viewer's locale
the locale preferencethe kit's per-principal preferencesdefault from the device, switched live
{ namespace, key, params }object state and noticesa message as data, rendered by each client
+
+ +

Speaking Every Language · proposal · 2026-11-02

+
+ + diff --git a/packages/chord/docs/next/layer-1.md b/packages/chord/docs/next/layer-1.md index 505fe7294..0129181df 100644 --- a/packages/chord/docs/next/layer-1.md +++ b/packages/chord/docs/next/layer-1.md @@ -14,7 +14,7 @@ checked, what a principal key is, what an error is. Pure functions, types and we Source: `packages/chord/src/next/contracts/` (declarations) and `packages/chord/src/next/data/` (per-call checks, principals, errors), split into files by concept. Everything else under `src/next/` is the frozen old kernel: never imported, never edited. Reused: `src/context/` (`Context`, `BACKGROUND_CONTEXT`, `context.telemetry`) and -`src/json.ts` (`JsonValue`), both unchanged. Schemas are TypeBox 1.3 (`typebox` in `node_modules`). +`src/json.ts` (`JsonValue`), both unchanged. Schemas were TypeBox 1.3 when L1 was built; since the schema switch (2026-11-01) they are pi-schema's plain JSON Schema (design §3 "Values"). The TypeBox details below are historical. ## 1. Values diff --git a/packages/chord/docs/next/p3-features/README.md b/packages/chord/docs/next/p3-features/README.md new file mode 100644 index 000000000..c7fd1d2e0 --- /dev/null +++ b/packages/chord/docs/next/p3-features/README.md @@ -0,0 +1,6 @@ +# p3 features (meeting prep) + +`book.md` references screenshots and clips as `{{img:|caption}}` / `{{clip:|caption}}`; they were captured +from p2 on 2026-11-02 (kept outside the repository: `/tmp/p3-prep/shots/` on slayer). Build: inline them with +`/tmp/p3-doc/inline.py book.md book.inlined.md`, then the explainer build. Published privately on Radius +(01m6efr04nezms4zx32p7pp0w1). diff --git a/packages/chord/docs/next/p3-features/book.md b/packages/chord/docs/next/p3-features/book.md new file mode 100644 index 000000000..3bdfba676 --- /dev/null +++ b/packages/chord/docs/next/p3-features/book.md @@ -0,0 +1,478 @@ +--- +title: p3: Feature Set and Work Packages (prep for the design meeting) +cover: p3 / Features +kicker: Meeting prep · p3 on Chord v2 · 2026-11-03 +subtitle: What p2 does today (in pictures), what p3 should ship first, how hands, sync and the plugin API fit, and a first cut into work packages +stats: + - 28 | feature areas in p2 today + - 21 | model-facing tools in p2 + - 78 | coding-agent extension examples checked against the p3 plugin API + - 12 | proposed work packages +note: | + Written by the redesign session for the meeting on 2026-11-03, from three research passes over p2 + (`apps/p2` at `b7af55ace`), the coding agent's extension examples and p2's environments and Radius code + (`/tmp/p3-prep/features.md`, `extensions.md`, `envs.md`), the owner's notes and Armin's summary of the 2026-11-02 + meeting. Chord v2's slices 1-13 are built (the app kit included); slice 11 (web and TUI) is building. Everything + marked "proposed" is for the meeting to decide. +colophon: p3 Feature Set and Work Packages · meeting prep · 2026-11-03 +--- + +# Before we begin {.preface} + +p2 grew feature by feature on the phone: 33 plugins, about 65,000 lines, a web shell and a TUI. p3 is its rewrite on +Chord v2, with an app kit that already gives hosts, workers, HTTP with sessions and pairing, plugins with versioned +APIs, files and export. The meeting's job is to decide what p3 ships first, what waits, and how the work splits. + +Some of it was settled yesterday (Armin's summary): sessions sync to R2 through Radius by commit log, the session index +is a replicated Chord table, every host kind loads plugins through registries, hands connect proactively. The owner +added: voice waits; environments turn around into hands; continuing a local session on Radius is core; subagents and +handoff are in, artifacts are questionable; the plugin API stays small enough to evolve. + +## Ten things to know + +:::findings +1. **Proposed base:** sessions (with delete), the agent and conversations, subagents, history and handoff, code mode, + shell commands, models and login, prompt resources, workspace (read and review), file index, status and notices, + search through the index, the web shell and the TUI, hands, session sync and continue-on-Radius. +2. **Proposed later:** voice, terminals (need channels and hands), artifacts, port forwards, chat mode and guest + sharing, MCP and `tool_search`. +3. **Hands replace SSH-dialled daemons:** a long-lived daemon dials the server (or Radius), is enrolled once by a + pairing code, and carries p2's existing daemon protocol over one multiplexed WebSocket. +4. **The sync handshake needs a driver identity.** Two machines writing one session both push "from Radius's last + seq" and silently diverge; an epoch or lease per session (Chord's rule S7) closes it. +5. **The index has two writers** once a session continues on Radius; three options, one recommended (derive rows from + session stores). +6. **Tools in base:** `read`, `write`, `edit`, `bash`, `codemode`, `subagent`, `team`, `agent`, `conversations`, + `history_*`, location tools, `rename_session`, `send_file`; artifacts, ports and `reload_plugins` wait or change. +7. **One CLI, no environment variables:** `p3 serve`, `p3 tui`, `p3 hand`, `p3 owner-link`, `p3 install`; settings as + arguments and files, per the app kit. +8. **The plugin API v1 is seven clusters,** almost all already existing: Durable's extension, a `Conversation` + contract, input transforms, environments and models, questions, UI registries, objects for state. +9. **Litmus test:** all 78 coding-agent extension examples are expressible; Doom needs two small fixes (frame splitting + in the TUI bridge, a path for large plugin assets); streaming frames from a worker and raw terminal ownership do + not fit, by design. +10. **Twelve work packages,** with p3's skeleton on the app kit and the session/index/sync core first. +::: + +## Words used throughout + +A **hand** is a daemon on a machine that connects out to a p3 server or to Radius and offers execution (commands, +files, watching, terminals). A **driver** is the single host allowed to write a session's store at a time; a +**replica** follows its commits. An **epoch** numbers driver changes. The **index** is the table of session metadata +(title, status, idle state) the session list reads. The **app kit** is Chord's `startApp`/`startWorker` layer. + +# p2 in Pictures {#pictures lede="What each feature area looks like today, on the web, the phone and in the terminal."} + +All content is invented (a small shop project, the owner Dana, a guest), the models are a fake provider's, tools ran +for real on a demo repository. Stills are from the web on a desktop and a phone and from the TUI; the clips loop. + +## Sessions + +{{img:sessions-list-desktop.webp|The sidebar: sessions grouped by directory, subagent counts, footer buttons.}} +{{img:sessions-drawer-phone.webp|The drawer on the phone, a subagent expanded.}} {{img:sessions-create-phone.webp|A new session on the phone.}} +{{img:sessions-create-directory-picker.webp|The directory picker for a new session: environment, path, folders, recent.}} +{{img:sessions-options-sheet.webp|The session sheet: title, share, working directory, pin, archive.}} +{{img:tui-session-selector.webp|The TUI's session selector, subagents threaded under their session.}} + +## Conversations and the agent + +{{clip:clip-streaming.mp4|Sending a question: thinking, a read card, a table and code stream in.}} +{{img:conversation-thinking.webp|A turn with thinking expanded and read, write and edit cards.}} +{{img:conversation-edit-diff-card.webp|An edit card with its diff.}} +{{img:conversation-bash-card.webp|A bash card with its output, then the answer.}} +{{img:conversation-streaming.webp|While an answer streams: the busy dot, the Steer composer, the stop button.}} +{{img:conversation-transcript-phone.webp|The transcript on the phone.}} {{img:conversation-model-picker-phone.webp|The model sheet on the phone.}} +{{img:conversation-cost-sheet.webp|The cost sheet: totals, cost and cache hits per response.}} +{{img:tui-transcript-tools.webp|The same agent in the TUI: thinking, tools, the answer, editor and footer.}} + +## Subagents, history and handoff, code mode + +{{img:subagents-spawn-and-report.webp|A spawned subagent in the parent's transcript, and its report.}} +{{img:subagents-conversation.webp|The subagent's own conversation, selected in the sidebar.}} +{{img:history-handoff-card.webp|A handoff: the continuation text, then a new context.}} +{{img:history-search-card.webp|history_search with its hits.}} +{{img:codemode-card.webp|A code mode script with three nested tool calls.}} + +## Shell commands and chat mode + +{{img:shell-command-tui.webp|A shell command run from the TUI with its output (the web can only show it).}} +{{img:chatmode-two-people.webp|Chat mode: owner and guest talk, an @pi mention gets a digest and an answer.}} + +## Sharing, devices, providers + +{{img:sharing-sheet.webp|The share sheet: read-only and read/write links, a connected guest.}} +{{img:sharing-join-page.webp|Joining through a share link.}} {{img:sharing-guest-view-phone.webp|The session as the guest sees it.}} +{{img:devices-sheet.webp|The devices sheet (the one-time token masked).}} +{{img:providers-login.webp|A provider's login methods.}} + +## Workspace, @ completion, terminals + +{{clip:clip-workspace-review.mp4|Changes, a diff, a line comment, Send all; the agent answers the review.}} +{{img:workspace-files.webp|Files: tree, sizes, upload.}} +{{img:workspace-changes.webp|Changes: branch, uncommitted files, commits.}} +{{img:workspace-search.webp|Text search across the project.}} +{{clip:clip-at-completion.mp4|@ completion in the composer.}} +{{img:terminals-web.webp|A web terminal panel with tabs and a key row.}} +{{img:tui-slash-changes-diff.webp|The TUI's /changes diff (read-only).}} + +## Environments, artifacts, attachments, search + +{{img:environments-panel.webp|The environments sheet: SSH environments and the ports entry.}} +{{img:artifacts-preview.webp|An artifact in the sandboxed preview.}} +{{img:attachments-message.webp|A message with an image attachment.}} +{{img:search-results.webp|Sidebar search: message hits for "rounding".}} + +## Status, notices, logs, voice, ports + +{{img:status-footer-viewers.webp|The composer footer: model, viewers, cache hit rate, cost, context.}} +{{img:notices-owner-notice.webp|An owner notice after a worker was killed, with Show logs.}} +{{img:logs-sheet.webp|The logs sheet following the server's log.}} +{{img:voice-sheet.webp|The voice sheet (later in p3).}} +{{img:ports-panel.webp|The ports panel without Radius (later in p3).}} + +## The web shell and the TUI + +{{clip:clip-phone-sheets.mp4|The phone: drawer, switching sessions, the session sheet, the share sheet, Back.}} +{{img:webshell-phone-sheet-2-directory.webp|The sheet stack on the phone: a second sheet pushed on the first.}} +{{clip:clip-tui-palette.mp4|The TUI's F1 palette: filter, run a command, the model selector.}} +{{img:tui-slash-commands.webp|/ command completion in the TUI.}} +{{img:tui-theme-picker.webp|The TUI's theme picker with preview.}} {{img:tui-theme-light.webp|The light theme.}} + +Not shown: clone and fork (TUI only, not run), SSH environments connected (no network in the capture), Radius and +public ports (off), voice in use (no microphone), dev mode reload pills, provider OAuth flows. + +# p2 Today, and the Proposed Cut {#features lede="Every feature area, what it is, and where it goes in p3."} + +Sizes are non-test lines in p2. "Base" means p3's first release; "later" means designed for, built after; "drop" means +not carried over. + +| Area | What it does | p2 size | Proposed | +| --- | --- | ---: | --- | +| Sessions | list by machine and directory, create, rename, pin, archive, relocate, clone, fork, unread, automatic titles | 2,186 | **base**, plus delete and in-session branches (`/tree`); clone and fork in the web too | +| Conversations and agent | transcript, send, steer, follow-up, outbox, stop, model and thinking, cost, compaction | 3,682 + web 2,700 | **base** | +| Subagents, one-shot agents, cross-session messages | named background agents, `team`, `agent` from scripts, `conversations` | 2,182 | **base** (owner: "awesome") | +| History and handoff | re-read earlier entries after compaction, hand off with a continuation | 533 | **base** | +| Code mode | scripts calling tools as nested durable calls | 155 + Durable | **base** (small, Durable owns it) | +| Shell commands | `!cmd` and `!!cmd` in the conversation's environment | in chat | **base**, web too (TUI only today) | +| Chat mode | people talk without a turn, `@pi` mentions | 876 | later | +| Sharing and guests | share links, guest names, revocation | 1,047 | later (the app kit's users and invites cover the start) | +| Devices and owner login | owner links, device cookies | 684 | **base**, replaced by the app kit's sessions and pairing | +| Models, credentials, provider login | picker, scope, API keys, OAuth flows | 3,828 | **base** | +| Settings | `settings.json`, hot reload | 87 | **base** | +| Prompt resources | AGENTS.md, skills, templates, read in the session's environment | 958 | **base** | +| Workspace | files, git status, diffs, search, review comments | 2,655 + web 1,341 | **base** (read and review; uploads later) | +| File index and `@` | ranked file completion | 242 | **base** | +| Terminals | detached PTYs, ghostty-vt in the web | 3,518 | later (channels, hands) | +| Environments and locations | SSH environments, directory picker, agents moving themselves | 2,546 | **base, redesigned as hands** | +| Artifacts and files | attachments, `send_file`, versioned artifacts with previews | 1,713 | attachments and `send_file` base (app kit `Files`); artifacts later | +| Search | sidebar search over titles and messages (FTS5) | 775 | **base** over the index; message text search through the same sync mechanism later | +| Status and presence | busy, tasks, usage, viewers, worker failures | 566 | **base** (feeds the index) | +| Logs and notices | owner log viewer, notices | 487 | **base** | +| Voice | rooms, barge-in, speech service | 3,770 + Python | later (owner) | +| Radius relay and port forwards | brain WebSocket, up to 50 forwards | 2,000 | relay **base** (as the hands/sync link), forwards later | +| Web shell | PWA, sheets, navigation, reconnect | 9,252 | **base**, rebuilt on slice 11 ("look and feel great") | +| TUI | launcher, sandbox, 40 commands, themes, keybindings | 6,720 + launcher | **base** on slice 11 | +| Dev and reload | supervisor, dirty marks, reload tool | 468 + scripts | **base** as the app kit's "apply changes"; the supervisor goes | +| Fake provider, `PI_MOBILE_*` aliases, `P2_DEV_RESUME` | test and legacy leftovers | 218 | drop | +| `tool_search`, MCP | not installed in p2 | - | later | + +Rough edges p3 should not inherit: features only in the TUI (shell commands, clone, fork, compact, new context), no +session delete, terminals and voice without a TUI side, docs that drifted from code (voice routes, TUI notes), the +plugin loader's module leak on reload (solved by Chord's plugin component). + +# Hands {#hands lede="Environments turned around: the machine connects to the server, not the other way."} + +## What p2 does + +p2's server and workers start a daemon (`pi-env`) over SSH on demand, one connection per session and per use (four +connection pools: each worker, the directory picker, the connection test, the terminal host). The daemon is +stateless and kills its commands 30 seconds after its connection goes quiet. Its frame protocol (exec, files, watch, +file listing, PTYs) does not depend on SSH. + +## What a hand is (proposed) + +| | p2's daemon | hand | +| --- | --- | --- | +| who dials | the server, over SSH | the hand, over WSS, to the server or to Radius | +| needs | sshd, a key, a network path to the machine | outbound HTTPS only | +| lifetime | per connection | a service (systemd, launchd, Termux:Boot), reconnecting | +| install | deployed and verified at each start | installed once, signed updates | +| machine identity | SSH host key | a hand key pair, enrolled once | +| connections | one per session and use | one per hand and server, multiplexed | + +- **Enrollment:** `p3 hand enroll ` shows a code; the owner approves it in the UI (the app kit's device + pairing); the server pins the hand's public key; fingerprints shown on both sides. Revocation in a Hands panel. +- **Reachability:** a hand reaches the owner's own server directly (LAN, own tunnel) or through Radius, which relays + hands to whichever host drives a session (the upsell: no tunnel to set up; Radius can also offer sandboxed machines + and hands of its own). +- **Protocol:** p2's daemon protocol over the WebSocket, unchanged in shape; two changes: several sessions share one + connection (cancellation, handles and backpressure per session), and a disconnect starts a grace period instead of + killing everything (detached commands and PTYs reattach on reconnect; this moves the terminal host's job into the + hand). +- **Binding:** a session's location becomes `{ hand, cwd }`; one server object holds the hand connections and the + `HarnessEnvironments` entry resolves a hand id to an execution environment; workers reach hands through the server's + links. An SSH machine stays possible as a hand kind the server dials. +- **Offline:** an operation needing an offline hand fails with a clear error the model sees; the hand's status shows + in the session list. + +# Session Sync and the Index {#sync lede="What was decided yesterday, and the three gaps it leaves."} + +## Decided + +The worker updates the session index from events and sends it to the server; sessions sync to R2 through Radius: +Radius reports per session its last sequence number, pi uploads the commit log from there; R2 holds the SQLite file +and the commit log, a SQLite Durable Object applies them; sessions have an owner id; the index is a Chord table +replicated like a session, its row updated only on metadata, session idle and subagent idle changes. + +Chord v2 has the pieces: a versioned outbox written in each commit's transaction, export archives, batch apply that +refuses gaps and divergence, export and import per address (app kit). Missing (the sync layer): upload and trim, +epochs, pull-back after a power loss, the split, starting from an archive. + +## Gap 1: who is the driver + +Two machines both write session X (the laptop resumed it while Radius continued it). Both ask Radius for its last +seq, both upload from there; Radius applies whichever arrives first and refuses the other as "not following". The +copies have now diverged and nobody knows. + +**Proposed:** a driver epoch per session, handed out by Radius. An upload names its epoch; an old epoch is refused; +the refused copy becomes a new session (Chord's rule S7: a split, never a rollback, because tools already ran). +"Continue on Radius" is an epoch change: the local driver waits for idle, uploads the tail and the session's files, +Radius bumps the epoch and starts the agent from its store, the local side becomes a replica or closes. Back is the +reverse. + +## Gap 2: two writers of the index + +When a session continues on Radius, Radius writes its index row, and so does the local server. + +| Option | How | Cost | +| --- | --- | --- | +| a. one index per writer | each host keeps its own index, Radius merges per owner, newest epoch wins per row | a second merge rule | +| b. rows derived from session stores | every host computes a session's row from the session's own commits | no index sync at all; each host needs the session's commits | +| c. rows tagged `{ epoch, seq }` | Radius keeps the highest per row | rows must carry their source position | + +**Recommended:** (b) where the session's commits are present (Radius has them all), with (c)'s tag on the rows a local +server writes for sessions it drives, so a list without the session store still sorts correctly. + +## Gap 3: what else moves + +Files outside the store (uploads, sent files, the app kit's `Files`) must sync as blobs before the batches naming +them, R2 as their home. Model access and secrets for a session continued on Radius: Radius's own models, or the +owner's provider keys stored on Radius (decide). Local prompt resources (the device's `AGENTS.md`, skills) must be read +through a hand or synced. Closed sessions have no worker: the server reads a session's outbox without starting it, or +starts it to sync. + +## The sync example + +`packages/chord/test/next/examples/60-session-sync/` (uncommitted, in `/home/badlogic/pi-sync-ex`) plays the decided +protocol with Chord's real store pieces (the outbox, `applyBatch`, export archives) and a small in-memory stand-in for +Radius. Its output, step by step: + +```text +1. laptop s1: lastSeq 4 outbox floor 0 +2. sync: uploaded 1..4, Radius at 4, trimmed through it +3. laptop s1: lastSeq 6 - sync: uploaded 5..6, Radius at 6, trimmed through it +4. laptop s2: lastSeq 3 outbox floor 3 - sync: archive uploaded (737 bytes), Radius imported it at 3 + laptop s2: lastSeq 4 - sync: uploaded 4..4 +5. Radius takes over, epoch 2; Radius writes seq 7 + stale laptop: lastSeq 7 - push: refused for an old epoch: this copy is now s1~split@6 +5b. no epoch check - stale push: refused: does not follow seq 7 + stale sync: nothing to send, Radius at 7 - outbox floor 7 (its own commit 7 trimmed) + stale: ... 5 assistant: (stale laptop) added a different test + Radius: ... 5 assistant: (on Radius) added a test (same seq, different data) +6. laptop takes epoch 3, pulls 7..7: same values: true; its next commit syncs at epoch 3 +``` + +:::sequence animate | The decided handshake (steps 1-3), then continuing on Radius (5) and back (6). +actor laptop "laptop" "driver, epoch 1" accent +actor radius "Radius" "replica per session" +laptop -> radius: which sessions do you have? +radius --> laptop: s1 at seq 4, epoch 1 +laptop => radius: commits 5..6 (epoch 1) +radius --> laptop: s1 at seq 6 (acknowledged: trim through 6) +divider "continue on Radius" +radius -> radius: epoch 2, Radius drives, writes seq 7 +laptop -> radius: commit 7 (epoch 1) +radius --> laptop: refused: old epoch (the copy splits) +divider "back to the laptop" +laptop -> radius: take epoch 3 +laptop => radius: pull 7..7, apply +::: + +What it shows: the handshake works and only the tail moves; a session Radius lacks starts from an archive; without +epochs a second writer's commit is lost silently (5b), with them it splits; taking a session back is a pull plus an +epoch. One more finding: `applyBatch` refuses while a host holds the store open, so the sync layer stops the session's +worker, applies, restarts it (decided 2026-11-02: usually nothing runs at that moment). The example lists nine things +its stand-ins fake that the sync layer must build (authenticated handshake, persisted epochs, the batch wire format, +retries, streaming bootstrap with files, the DO and R2 side, the continue fence, the split's new id, pull-back after a +power loss). + +# Branches and Subagent Context {#branches lede="pi's /tree, fork and clone, and subagents that start from the parent's context."} + +pi branches three ways; p2 has two, in the TUI only: + +| pi | What it does | p2 | +| --- | --- | --- | +| `/tree` | branch inside the session: pick an earlier message, edit and resubmit; the old branch stays; leaving a branch can summarize it into the one entered | missing | +| `/fork` | a new session from an earlier user message | TUI only, main conversation only | +| `/clone` | a copy of the active branch as a new session | TUI only | + +Durable already has the building block: `conversation.fork(at)` creates a conversation in the same session store that +shares every entry up to `at`, without copying. Proposed for p3: + +- **Base:** editing an earlier message forks the conversation there (an in-session branch); a branch list switches + between them; fork and clone to a new session; all in the web and the TUI. +- **Later:** a full tree navigator and branch summaries. +- **Index and sync:** branches are conversations inside one session store, so sync carries them for free; the index + marks the active branch and tells branches from subagents. + +**Subagents with the parent's context** (owner, 2026-11-02): `subagent({ action: "spawn", ..., context: "inherit" })` +forks the parent conversation at the last entry before the assistant message holding the spawning call. The child +knows everything the parent knew; its model, tools and location may differ; with the same model the shared prefix is +usually a prompt-cache hit, otherwise it is paid in full (the tool's description says so). Base in p3. + +# Tools Built In {#tools lede="p2's 21 model-facing tools, and which ship with p3."} + +| Tool | p2 plugin | Proposed | +| --- | --- | --- | +| `read`, `write`, `edit`, `bash` | coding-tools (Durable) | base | +| `powershell` | powershell | base (deferred tool, Windows hands) | +| `codemode` | codemode | base | +| `agent` (from scripts) | agent | base | +| `subagent`, `team` | subagents | base | +| `conversations` | conversations | base | +| `history_search`, `history_read`, `history_handoff` | history | base | +| `location_envs`, `location_dirs`, `location_move` | locations | base, renamed around hands (`hands`, `hand_dirs`, `move`) | +| `rename_session` | sessions | base | +| `send_file` | send-file | base (on the app kit's `Files`) | +| `artifact` | artifacts | later (owner: maybe not as useful) | +| `ports` | ports | later (with forwards) | +| `reload_plugins` | dev-tools | base as an "apply changes" tool for the owner, not tied to a checkout | +| `tool_search` | Durable | later (with MCP) | + +Every tool is a plugin in the app's own root, so any of them can be removed, replaced or added without touching base +code (the app kit's rule: base code is the API, bootstrap and a start-data facet). + +# Running p3 {#running lede="Starting the server, the CLI, themes and keybindings."} + +## The CLI (proposed) + +One binary, `p3`, and no environment variables (app kit decision); settings are arguments or files in the data +directory. + +| Command | Does | +| --- | --- | +| `p3 serve [--data-dir ] [--host ] [--port ]` | starts the server (`startServer`); prints the one-time owner link | +| `p3 owner-link` | prints a new owner link for a running server (through the owner token file) | +| `p3 tui [url] [--pair]` | the sandboxed TUI; on the same machine it uses the owner token file, elsewhere device pairing | +| `p3 hand enroll `, `p3 hand run` | enrolls this machine as a hand, runs it (or installs a service) | +| `p3 radius login` | signs the server into Radius (device code) | +| `p3 install ` | installs a plugin into the user's root | +| `p3 export `, `p3 import ` | the app kit's `exportData`/`importData` (a running session is not stopped) | + +The data directory defaults to `~/.p3`. Its files: `settings.json`, `models.json`, `auth.json` (locked), +`AGENTS.md`, `skills/`, `prompts/`, `plugins/` (the user's root), the server store, a directory per session worker, +`owner-token`, `logs/`. Plugin-specific configuration moves into plugin objects (documents) instead of more files. + +## Themes and keybindings + +Themes follow the looks-and-themes proposal (decided 2026-11-02): design tokens as CSS variables on the web, roles in +the TUI, one `Themes` registry plugins contribute to, one dark and one light choice per user. Keybindings: commands are +registry entries with default keys (p2's `TuiCommands` with `when` conditions, 78 binding ids); the user's overrides +live in a preferences document (per user, synced like any data), not in files on each client; the web gets the same +command registry with its own key handling. + +# The Plugin API {#api lede="The smallest surface that still expresses everything the coding agent's extensions do."} + +## The rule + +Offer few things, each stable: what an API offers, it must keep (versioned in the import path, `@p3/app/v1`). Every +cluster below is a registry or a contract that already exists in Chord, Durable or p2; p3 v1 publishes those and +nothing else. Legacy concepts of the old extension API are not carried: CLI flags, an event bus, RPC-mode UI, `hasUI` +checks, "destructive switch" gates, `pi.exec` running locally regardless of the environment. + +## Seven clusters + +| Cluster | Host | Mechanism | Covers | +| --- | --- | --- | --- | +| A. The agent's turn | worker | Durable's `Extension` in `HarnessExtensions`: tools, `beforeTool`/`afterTool`, `beforeRequest`, sections, wraps, `onYield`, `afterRun`, `beforeCompact` | 20 examples: hello, permission-gate, protected-paths, tool-override, plan-mode, todo, custom-compaction, ... | +| B. Conversation control | worker object | a `Conversation` contract: `submit`, `configure`, `append` (typed entries), `compact`, `fork`, `rename`; state: busy, agent, usage | send-user-message, file-trigger, session-name, preset, handoff, qna, ... | +| C. Input | worker | `InputTransforms` registry (rewrite or answer input before the model) | input-transform, inline-bash | +| D. Environments and models | worker, server | `HarnessEnvironments` (ssh, gondolin, hands as kinds), `HarnessModels` (providers, virtual models), `LoginProviders` | ssh, gondolin, custom providers, jev-router | +| E. Asking the user | server object, UI renderers | `Questions.ask` from worker code; `QuestionKinds` per UI kind | permission-gate, question, questionnaire, timed-confirm, project-trust | +| F. UI | tui, web | registries: `Commands` (slash, palette, keys), `ToolRenderers`, `EntryRenderers`, `Slots` (header, footer, status, widgets), overlays through the UI context, editor extension points | every UI-only example, the games | +| G. State and standing behaviour | any | `local` objects owning natives, documents for state | file-trigger's watcher, gondolin's VM, game saves | + +```ts +// a plugin with a tool, a gate and a renderer: plugins/todo/ +// worker.ts +self.contribute(HarnessExtensions, { name: "todo", tools: [todoTool], hooks: [hook(ToolTask, { beforeTool: guard })] }); +// tui.ts +self.contribute(TuiToolRenderers, { todo: { call: (args) => todoCall(args), result: (r) => todoList(r.details) } }); +self.contribute(TuiCommands, { "todo.show": { slash: { name: "todos" }, run: (ui) => ui.overlay("todos", new TodoView(ui)) } }); +// web.tsx: the same registries with React components +``` + +## The litmus test + +All 78 examples map onto the seven clusters (table in `/tmp/p3-prep/extensions.md`). The examples that mix agent and +UI split into a worker facet and a tui/web facet sharing a contract file: a tool that wants a dialog asks the +`Questions` object, which works from any client, several clients or none. + +- **Snake, Space Invaders, tic-tac-toe:** a command plus an overlay in `tui.ts` and a canvas in `web.tsx`, engine in a + shared file, saves in a user-keyed document. Fit today. +- **Doom:** fits with two fixes in the TUI: the sandbox bridge caps a frame at 64 KiB and a Doom frame is about 160 KB + (split writes at escape boundaries), and the sandbox has no files or network, so large plugin assets (a 4 MB WAD) + need a documented "asset of my plugin" path (chunked reads through the plugin's server facet). +- **Does not fit, by design:** streaming 35 FPS frames from a worker to clients (objects carry state; needs channels), + engine state surviving an in-place reload (needs an explicit save), raw terminal ownership (`interactive-shell` + becomes a terminal view; terminal notifications need a launcher action). + +## What v1 leaves out + +Editor replacement (extension points only), raw provider stream events and response headers, a fork hook, a +`turn_start` hook, an event bus. Each has one or two examples; add when a real plugin needs it. + +# Work Packages {#packages lede="A first cut, in dependency order, for the meeting to rearrange."} + +| # | Package | Contains | Depends on | +| --- | --- | --- | --- | +| 1 | p3 skeleton | `startServer` with p3's API v1 modules (shared, web, tui, worker), core plugins in the app's root, `p3` CLI | Chord slices 11-13 | +| 2 | Sessions and conversations | Durable on the Chord store, session lifecycle with delete, the conversation contract, subagents, history and handoff, code mode, shell commands | 1 | +| 3 | Session index | the index table, worker-to-server updates on metadata and idle changes, the session list, search over it | 2 | +| 4 | Session sync to Radius | outbox upload and trim, epochs, pull-back, split, files as blobs, continue on Radius and back | 2, 3, Radius side | +| 5 | Radius side | per-session SQLite DO, commit log and SQLite on R2, epochs, the sync handshake, hand relay | with 4 | +| 6 | Hands | daemon as an outbound service, enrollment by pairing, multiplexed protocol, reattach after disconnects, server hand pool, `HarnessEnvironments` kind, SSH as a hand kind | 1 | +| 7 | Web shell | p2's layout and flows rebuilt on slice 11 ("look and feel great"), themes | 1, slice 11 | +| 8 | TUI | p2's TUI on slice 11's sandbox, frame splitting, plugin assets path, keybindings and themes | 1, slice 11 | +| 9 | Models and settings | catalog, credentials, provider login flows, settings, prompt resources | 1 | +| 10 | Workspace | files, git, diffs, search, review comments, file index and `@`, through hands | 6 | +| 11 | Plugin API v1 and docs | the seven clusters published in `@p3/app/v1`, authoring guide, ported examples as the litmus (hello, permission-gate, todo, plan-mode, snake, doom) | 1, 2, 7, 8 | +| 12 | Access and status | owner, devices and pairing (app kit), notices, logs, status and presence | 1 | + +Later: voice, terminals (channels and hands), artifacts, port forwards, chat mode and guest sharing, MCP and +`tool_search`. + +# Open Questions {.epilogue lede="To decide in the meeting."} + +1. The proposed cut: base versus later per area (first table). +2. Driver identity: epochs per session, handed out by Radius, with splits on conflict? +3. The index with two writers: derived rows plus `{ epoch, seq }` tags, or one of the other options? +4. Who syncs a closed session (the server reading its outbox, or starting its worker)? +5. Files outside the store as blobs on R2, uploaded before their batch? +6. Models and secrets for sessions continued on Radius: Radius's own models, or the owner's keys stored there? +7. Hands: through Radius as a relay to the driver, and may one hand serve the local server and Radius at once? +8. Do hands keep commands and terminals alive across disconnects (absorbing p2's terminal host)? +9. Hand install and updates: signed binary, service install, version negotiation. +10. Is SSH kept as a hand kind for machines without a service? +11. Plugin API v1: the seven clusters as listed, and what is explicitly left out? +12. Work package owners and order. + +# Sources {.appendix} + +| Source | What | +| --- | --- | +| `/tmp/p3-prep/features.md` | p2's feature areas with files, sizes, tools, rough edges; the CLI, environment variables and data files | +| `/tmp/p3-prep/extensions.md` | all 78 extension examples, the capability matrix, the clusters with sketches, the hard cases | +| `/tmp/p3-prep/envs.md` | p2's environments and Radius link, Chord's sync pieces, the sync and hands sketches, open questions | +| `apps/p2/docs/p3-functions/` | the functionality map (partly outdated) | +| `packages/chord/docs/next/` | design, decisions (2026-11-02 meeting summary), app kit and plugin model, theming | diff --git a/packages/chord/docs/next/plugin-authoring.md b/packages/chord/docs/next/plugin-authoring.md new file mode 100644 index 000000000..64c2063b4 --- /dev/null +++ b/packages/chord/docs/next/plugin-authoring.md @@ -0,0 +1,176 @@ +# Writing a Chord plugin + +For agents and people writing plugins for an app built on Chord v2. Self-contained: everything a plugin author needs +is here. In the examples the app is "Acme", its API package `@acme/app`; replace it with your app's (its docs name the +package and the versions it provides). + +## What a plugin is + +A plugin is a feature of a running app: it serves objects (state plus methods) and contributes to the app's +registries (routes, panels, tools). The app loads it on its hosts (the server, its workers, browser pages, terminal +clients), reloads it when it is edited, and leaves it out alone when it is broken. Plugins talk to each other only +through object contracts and registries. + +## Layout + +A **directory plugin**, in one of the app's plugin roots: + +```text +plugins/stats/ + package.json { "name": "@bob/stats", "dependencies": { ... } } required; the name is the plugin's + server.ts the server facet one file per host kind the app has, each optional: + worker.ts the worker facet .ts, .tsx, .js or .mjs; default export: a facet + web.tsx the web (page) facet + tui.ts the terminal client facet + contract.ts contracts every kind may import (data only) + contract.web.tsx contracts and tokens for web facets only (`contract..ts(x)` per kind) + *.ts, *.css, assets/ plain files of the plugin +``` + +- `package.json`: `name` (an npm package name, scoped or not) is the plugin's name; `dependencies` are bundled into + its scripts; `"chord": { "external": ["better-sqlite3"] }` keeps named packages external in Node facets (native + addons). +- A **package of several plugins** (a repository holding them) lists their folders in its own `package.json`: + `"chord": { "plugins": ["./notes", "plugins/*", "!plugins/draft"] }` (paths, globs and `!` exclusions; `.` is the + package itself; a listed folder inside another listed one is refused). Without the field, the package is one plugin + when its top has facet files, else **every** subfolder with a `package.json` is a plugin: a repository with nested + packages that are no plugins (a helper library, examples, fixtures) must list its plugins in `chord.plugins`. +- A plugin with no facet file holds contracts others import. +- A **single-file plugin** is a file at a root's top named `..ts` (or `.tsx`, `.js`, `.mjs`): + `hello.server.ts` and `hello.web.tsx` are the plugin `hello` with two facets. No `package.json`, so no + dependencies; it imports only the app's API and defines its contracts inline. When it needs more, make it the + directory `hello/` with a `package.json` named `hello`: the app treats it as the same plugin. + +## A facet + +```ts +// plugins/stats/worker.ts +import { defineFacet, int, local, str } from "@acme/app/v3"; +import { Indexers } from "@acme/app/v3/worker"; +import { Notes } from "../notes/contract.ts"; + +const Stats = local("bob.stats", { version: 1, params: { project: str() }, members: { words: int() } }); + +export default defineFacet((host, self) => { + self.contribute(Indexers, { name: "word count" }); // a registry entry while the plugin is loaded + host.serve(Stats, ({ project }, self) => { // a factory, run per object + const notes = self.get(Notes, { project }); // a dependency: Stats waits until Notes is live + return { words: self.derive([notes.text], ([text]) => text.split(/\s+/).filter(Boolean).length) }; + }); +}); +``` + +A facet only declares (`host.serve`, `self.contribute`, `host.references`), synchronously. Everything at runtime +happens in objects: state in documents and tables, standing behaviour in a `local` object with empty params (it lives +as long as the host). Names are prefixed by the plugin (`bob.stats`). + +## What a plugin may import + +| Import | Resolves to | +| --- | --- | +| its own files (`./label.ts`, `./icon.svg?url` in web facets) | inlined into its script | +| the app's API, a version in the path: `@acme/app/v3`, its kind's module `@acme/app/v3/worker` | the app's own instance (a host package), never copied | +| another plugin of the same root (one repository): `../notes/contract.ts` | inlined (a copy, frozen at this build) | +| a package in `dependencies`: `@bob/notes`, `@bob/notes/web` | inlined from `node_modules` at the installed version, through the package's `exports` | +| a plugin inside a dependency on a whole repository of several: `@bob/plugins/notes/contract.ts` | inlined, as any package file | +| a `chord.external` package, a Node builtin (Node facets) | loaded when the script runs | +| `pluginFile` from `@earendil-works/chord/next/plugins` | the absolute path of a file of this plugin | + +Refused at build, each with the file, line and rule: + +- `@acme/app/v1` when the app does not list v1 ("@acme/app/v1 is not available: this app provides v3, v2"); + another kind's module (`@acme/app/v3/web` from a worker facet); two API versions in one facet. +- Chord itself (`@earendil-works/chord/next`): the app's API re-exports what plugins may use. +- A relative path leaving the plugin's root (base code, another root); a file of a directory that is no plugin. +- A file copied from another plugin importing anything but the API and its own plugin's files. +- A per-kind file (`contract.web.tsx`, `contract.Web.ts` alike) or another kind's facet file (`web.tsx`) reached from + a facet of another kind, directly or through other files (the error names the chain). Files inside npm packages are + checked only when the package is plugin code (its `package.json`, or one above it in the package, has a `chord` + field); ordinary libraries name files as they like (`pdf.worker.mjs`). +- A package file whose real path is outside the plugin's own `node_modules` (a `file:` dependency linked elsewhere, a + package importing `../../src`). +- A copied file importing a Node builtin or a package. +- A package not in `dependencies`; `import.meta`, `__dirname`, `__filename`, a literal `import("x")`, top-level + `await` (read files through `pluginFile`). +- In a single-file plugin, anything but host packages. + +Contract files mention native things (React components, functions) only through `import type`; at runtime they import +only the API and the plugin's own data-only files. + +## Contracts and versions + +A plugin carries the contract it was written against. Plugins changing together (one root, one repository) import +each other's contract files by relative path. A plugin from another repository takes the contracts as a dependency +pinned to a version: an npm version, or a git ref (`"@bob/notes": "github:bob/notes#"`, `"git+https://...#v2"`). +The app's own contracts for third parties come with its API (`@acme/app/v3` exports `Notes`). When a contract +changes shape, bump its `version`; the provider serves older consumers through `accepts`, else they wait "contract +mismatch" (visible, never wrong data). + +## How the app builds it + +The server builds each facet with one esbuild call from source: TypeScript and TSX (JSX with the automatic React +runtime), JS, JSON, CSS (web facets: one stylesheet per facet), assets and WASM by URL (`import url from "./x.svg?url"`). +No type check, no build hooks, no `dist` needed; generated files are committed. The script is evaluated in each +process of its kind with that process's API modules. A failed build or evaluation leaves the plugin out (at startup) +or keeps its old code (at a reload), reported with the error. + +## Styles and themes (web and terminal facets) + +- A web facet's CSS is served with its script and swapped when the plugin reloads. Scope it with a **CSS module**: a + file named `*.module.css` has its class names made unique to the plugin and the file, and its import is the mapping + (`import styles from "./notes.module.css";
`). Plain CSS files still work; their class + names should then start with the plugin's name. +- Read the app's look from its CSS variables (`var(--accent)`, `var(--bg)`, `var(--fg)`, `var(--muted)`, `var(--error)`, + `var(--warning)`, `var(--success)`, and the app's own roles), never fixed colours: the active theme sets them. +- In a terminal facet, ask the app's theme lookup for a role (`theme.fg("accent", "Notes")`); never write raw colours. +- A theme is data: contribute a `Theme` (`{ id, name, mode, colors, web?, tui? }`) to the kit's `Themes` registry, + from a web facet and a terminal facet to reach both. Users pick one dark and one light theme. + +## Installing + +- `plugins.installPlugin({ git: "https://github.com/bob/stats.git", ref: "v1.2.0" }, root, ctx)`: a shallow clone of + the branch, tag or commit into `/stats`; its plugins are the folders its `chord.plugins` lists, else by + convention. `{ npm: "@bob/stats", version: "1.2.0" }`: the package extracted into `/@bob/stats`. Then their + dependencies are installed (`npm install --omit=dev --ignore-scripts`; no lifecycle scripts run). Refused: a plugin + name the app has already, an existing directory, a ref or name starting with `-`. +- Removing is deleting the directory. To update, install another ref (a new directory) or delete and install again. + +## Testing a plugin alone + +```ts +import { testPlugin } from "@earendil-works/chord/next/plugins"; +import * as apiV3 from "@acme/app/v3"; +import * as apiV3Worker from "@acme/app/v3/worker"; + +const Notes = apiV3.local("acme.notes", { version: 1, params: { project: apiV3.str() }, members: { text: apiV3.str() } }); +const Stats = apiV3.local("bob.stats", { version: 1, params: { project: apiV3.str() }, members: { words: apiV3.int() } }); +const notesStub = apiV3.defineFacet((host) => host.serve(Notes, () => ({ text: apiV3.constant("a b c") }))); + +const t = await testPlugin("./plugins/stats", { + kind: "worker", + kinds: ["server", "worker", "web"], // the app's kinds (by default: those its files name) + api: { name: "@acme/app", versions: ["v3"] }, + modules: { "@acme/app/v3": apiV3, "@acme/app/v3/worker": apiV3Worker }, // the real API modules + stubs: [{ name: "@acme/notes", facet: notesStub }], // facets serving what it uses + principal: { kind: "worker", project: "p1" }, // the host's principal (optional) +}, ctx); +const stats = await t.host.get(Stats, { project: "p1" }, ctx); // stats.words.value: 3 +t.contributions(apiV3Worker.Indexers); // its entries in a registry nobody serves (the helper collects them) +t.unserved(); // contracts its requests wait for: stub these +await t.root(async (ctx) => ..., ctx, { principal: { kind: "user", id: "u1" } }); +await t.close(ctx); +``` + +The plugin is built as production builds it and loaded on an in-memory host (a `:memory:` store). Stubs are ordinary +facets with fake data. A test declares the contracts it reads again (contracts are data: the same name, version and +schema bind). + +## Checklist + +1. `package.json` with a valid name (or a single file `..ts`). +2. One facet file per kind it runs on; each default-exports `defineFacet(...)`. +3. Imports: the app's API at one version, own files, same-root contract files by relative path, other repositories' + by package name pinned in `dependencies`. +4. Contracts prefixed with the plugin's name; a version bump for every change of shape. +5. Files of one kind named `*..ts(x)`. +6. A test with `testPlugin` and stubs for every name `t.unserved()` lists. diff --git a/packages/chord/docs/next/plugin-model/book.md b/packages/chord/docs/next/plugin-model/book.md new file mode 100644 index 000000000..5717711a3 --- /dev/null +++ b/packages/chord/docs/next/plugin-model/book.md @@ -0,0 +1,186 @@ +--- +title: The Plugin Model: what a plugin is, what it may import, and how versions meet +cover: The Plugin / Model +kicker: A design proposal · Chord v2 · 2026-10-31 +subtitle: Plugins, base code, versioned APIs and contracts, in one place, as discussed with the owner on 2026-10-31 +stats: + - 4 | kinds of import a plugin may write + - 1 | integer per API version, in the import path + - 0 | package.json fields Chord reads besides name and dependencies + - 0 | type checks in the build +note: | + A proposal written by the redesign session after a day of discussion with the owner. It collects what was decided + and what is proposed into one model. Decided points cite their decision in `decisions.md`; proposed ones are marked. + Nothing here is built yet beyond slices 7c and 10. +colophon: The Plugin Model · proposal · 2026-10-31 +--- + +# Before we begin {.preface} + +An app built on Chord is mostly plugins. Its own features are plugins, users add plugins, and agents write plugins +while the app runs. A plugin uses the app (where things live, who calls, how to declare objects), uses other plugins +(their objects and extension points), and uses npm packages. Each of those can change on its own schedule: the app +gets a new version, a plugin it depends on changes its contract, a package publishes a fix. + +The questions the owner kept asking were all versions of one: when a plugin is written against one state of the world +and runs against another, what does it get, and who notices a mismatch? This model answers it with one rule for each +kind of import, and versions that sit where the dependency is written: in the code. + +## Eight things to know + +:::findings +1. **A plugin is a directory, or a single file.** A directory has a `package.json` (its name, its npm dependencies) + and one file per host kind; a single file `hello.worker.ts` is a plugin named `hello` with no dependencies. +2. **Base code is internal unless it is an API.** The app exposes only versioned API modules to plugins; everything + else is unreachable. +3. **API versions are in the import path.** `import { Session } from "@acme/app/v3"`: the plugin says what it was + written for where it imports it; an unavailable version is a build error. +4. **APIs are split by host kind.** A shared module holds what every host may use (contracts, registry tokens, host + kinds); `/v3/web` and `/v3/tui` hold UI pieces; a worker never loads React. +5. **Contracts carry their own versions.** A plugin carries the contract version it was written for; the provider + binds it if it accepts that version, else the plugin waits with a clear reason. +6. **A plugin imports four things:** its own files, host packages, its npm dependencies, and the app's own plugins by + relative path. Nothing else. +7. **Settings are arguments, extensions are registries.** What the app decides at startup is passed to its components; + what plugins add is contributed. +8. **No type checking in the build.** Types come from where imports come from; editors and agents check, the build + only builds. +::: + +## Words used throughout + +**Base code** is the app's own program; its **API** is the part plugins may import, published as **API modules**, +one per version and host kind. A **host package** is a module the running process provides to every plugin (Chord, +the app's API modules). A **contract** is an object type's declaration; a **registry token** an extension point's. +A **facet** is a plugin's declarations for one host kind. + +# What a Plugin Is {#plugin lede="A directory or a single file, named once, with one file per host kind."} + +```text +plugins/notes/ a directory plugin + package.json { "name": "@acme/notes", "dependencies": { "slugify": "1.6.6" } } + contract.ts the object types and registry tokens it offers + worker.ts server.ts web.tsx one facet per host kind it needs +plugins/hello.worker.ts a single-file plugin "hello" with one worker facet +plugins/hello.web.tsx the same plugin's web facet +``` + +A directory plugin's name is its `package.json` name; Chord reads only `name` and `dependencies` from it. A +single-file plugin's name is its file's base name; it has no dependencies and imports only host packages. When a +single file needs a package or a second file, it becomes a directory. Two roots with the same name: the first root +wins, the other is reported (slice 7c). + +# Base Code: Internal and External {#base lede="What the app keeps to itself, what it exposes, and how exposure is versioned."} + +The app's base code is everything that starts the process: settings, startup, core objects' implementations, the +session index, migrations. Plugins must not depend on any of it, or every refactoring of the app would break them. +So the app draws a line and exposes only its **API**: + +```text +src/api/v3/index.ts shared: host kinds, the principal type, defineFacet per kind, core contracts, registry tokens +src/api/v3/web.ts web only: shared UI pieces (a Sheet, theme variables), React-typed registry tokens +src/api/v3/tui.ts TUI only: its UI pieces +src/… internal: everything else +``` + +Each API module is a host package of the kinds that may use it: `@acme/app/v3` for every kind, `@acme/app/v3/web` for +the web kind only. A worker plugin importing `@acme/app/v3/web` is a build error, and a worker process never loads +React. The shared module stays free of runtime imports of UI libraries: a registry token that needs a React type names +it with `import type`, which the build erases. + +:::note | Proposed +The split of the external API into a shared module and per-kind modules answers the owner's worry about "web, TUI, +worker, server things all in the same index.ts". Proposed, not yet decided. +::: + +# Versions in the Import Path {#versions lede="A plugin says what it was written for where it imports it."} + +The app's API changes over time: a host kind is renamed, a principal variant changes. A plugin written for an older +API must not silently get the new one. So every API module has a version in its path, and the app lists which versions +it still provides: + +```ts +// plugins/stats/worker.ts +import { defineFacet, Session, Panels } from "@acme/app/v3"; +import { remote, str } from "@earendil-works/chord/next/v2"; +``` + +- The app provides `/v3`, its current API. If version 2 plugins still work unchanged against it, `/v2` is listed too, + pointing to the same module. An app that needs a compatibility layer makes `/v2` a thin module over `/v3`. +- An import of a version the app no longer lists is a build error: "@acme/app/v1 is not available: this app provides + v3, v2". The plugin keeps its old code, and the message tells an agent exactly what to change. +- Nothing to fill in elsewhere: no `package.json` field, no npm ranges. Skeletons and examples import the current + version, so an agent copies it without thinking. +- Chord's own surface is versioned the same way (`@earendil-works/chord/next/v2`). + +Decided: "the API version must be declared by the plugin in code" (owner, 2026-10-31); the import path is how. + +# Contracts and Registries {#contracts lede="Finer versions inside the API, and what crosses between plugins."} + +Contracts and registry tokens keep their own versions inside the API, because they evolve independently of the API as +a whole: + +- **An object type**: the consumer carries the contract version it was written for; the provider binds it at that + version if it accepts it (`accepts` maps arguments forward and results and state back), otherwise the consumer waits + "contract mismatch: v1, served v3". Shapes are fingerprinted, so a changed shape without a version bump is caught. +- **A registry token**: a name, a version and TypeScript types for what is contributed and read. Values are in-process + things (functions, components), which cannot be fingerprinted: bumping the version when they change is the defining + plugin's rule, and its `accepts` adapts older contributions. +- **Host kinds on registries** (proposed): a token names the kinds that serve it (`hosts: ["web"]`), typed, so a worker + facet contributing to a web registry is a type error and a load error, not a silent no-op. + +A contract file imports only host packages at runtime (Chord, the app's API); native types it mentions are type-only. + +# What a Plugin May Import {#imports lede="Four kinds of import, each with one rule."} + +| Import | Resolved to | Versioned by | +| --- | --- | --- | +| its own files (`./store.ts`) | copied into its script | the plugin itself | +| host packages (`@acme/app/v3`, `@earendil-works/chord/next/v2`, `react`) | the running process's instance, shared | the path's version; the app lists what it provides | +| npm packages in its `dependencies` (`slugify`, a third party's contracts package) | its own `node_modules`, bundled | the installed version | +| the app's own plugins (`../notes/contract.ts`), in the same root | copied into its script | the same revision: one repository | + +A third-party plugin that wants the app's plugins' contracts gets them through the app's API (core contracts) or the +app's published package, as an npm dependency at the version it was written for, never from the running plugin's +files. Anything else (a path into base code, an unlisted version, a package not in its dependencies) is a build error. + +# Settings and Extensions {#settings lede="How the app configures its components without letting plugins override it."} + +Decided ("Settings are arguments, extensions are registries"): what the app decides at startup is an argument to the +component's facet, which base code constructs, and plugins cannot change it: + +```ts +const http = createHttp(server, { listen: { port: 8788 }, links: { path: "/chord", maxPayload: MiB }, closeTimeout: 10_000, onError }, ctx); +``` + +What plugins add while the app runs is a registry: routes, authenticators, tools, panels. A component may offer both +for one thing: the session index takes its indexer as an argument, and may also follow a registry of indexers if +plugins should supply them. + +# Building and Evaluating {#build lede="Unchanged from slices 7c and 10: one build on the server, evaluation everywhere."} + +Only the server builds, because only the server has the sources. `createPlugins` builds every kind and needs only +names: per kind its platform and its host package names. Each process evaluates the scripts of its kind with its own +module objects (the server's at `plugins.load`, a worker's in `workerMain`, a page's in its base bundle), so a plugin +shares that process's Chord and API. Base-code facets are passed beside the plugins; they are not plugins, because +they close over the process's own values and never reload. + +# Types and Tests {#tools lede="Where types come from, and how a plugin is tested alone."} + +Types come from where imports come from: relative files and npm packages resolve natively in TypeScript; the app's API +modules ship `.d.ts` per version in the app's package. There is no type checking in the build (the owner: seconds per +build is too slow); editors check as an author types, and an agent may run `tsc` as a tool. + +Testing a plugin alone (proposed): `testPlugin(dir, { kind, stubs })` builds the plugin like production, loads it on an +in-memory host with the kind's real host packages, and loads stub facets that serve the contracts it depends on. A stub +is an ordinary facet; there is no mocking framework. + +# Open Questions {.epilogue lede="What is proposed here and still to decide."} + +1. The external API split by host kind (`/v3`, `/v3/web`, `/v3/tui`). +2. Host kinds on registry tokens, typed and checked at load. +3. Single-file plugins (`hello.worker.ts`, named by the file). +4. Contracts of the app's plugins for third parties: through the API modules only, or also the app's npm package. +5. `testPlugin` with stub facets. +6. Reversing "Plugins import each other by name" (2026-10-31, morning) in favour of relative paths within the app + and npm dependencies across it. diff --git a/packages/chord/docs/next/plugin-model/index.html b/packages/chord/docs/next/plugin-model/index.html new file mode 100644 index 000000000..2549955ac --- /dev/null +++ b/packages/chord/docs/next/plugin-model/index.html @@ -0,0 +1,310 @@ + + +The Plugin Model: what a plugin is, what it may import, and how versions meet + + + + +
+
+
A design proposal · Chord v2 · 2026-10-31
+

The Plugin
Model

+

Plugins, base code, versioned APIs and contracts, in one place, as discussed with the owner on 2026-10-31

+
4kinds of import a plugin may write
1integer per API version, in the import path
0package.json fields Chord reads besides name and dependencies
0type checks in the build
+

A proposal written by the redesign session after a day of discussion with the owner. It collects what was decided +and what is proposed into one model. Decided points cite their decision in decisions.md; proposed ones are marked. +Nothing here is built yet beyond slices 7c and 10.

+
+
+ + +
+

Before we begin

+

An app built on Chord is mostly plugins. Its own features are plugins, users add plugins, and agents write plugins +while the app runs. A plugin uses the app (where things live, who calls, how to declare objects), uses other plugins +(their objects and extension points), and uses npm packages. Each of those can change on its own schedule: the app +gets a new version, a plugin it depends on changes its contract, a package publishes a fix.

+

The questions the owner kept asking were all versions of one: when a plugin is written against one state of the world +and runs against another, what does it get, and who notices a mismatch? This model answers it with one rule for each +kind of import, and versions that sit where the dependency is written: in the code.

+

Eight things to know

+
    +
  1. A plugin is a directory, or a single file. A directory has a package.json (its name, its npm dependencies) +and one file per host kind; a single file hello.worker.ts is a plugin named hello with no dependencies.
    2. +
  2. Base code is internal unless it is an API. The app exposes only versioned API modules to plugins; everything +else is unreachable.
    3. +
  3. API versions are in the import path. import { Session } from "@acme/app/v3": the plugin says what it was +written for where it imports it; an unavailable version is a build error.
    4. +
  4. APIs are split by host kind. A shared module holds what every host may use (contracts, registry tokens, host +kinds); /v3/web and /v3/tui hold UI pieces; a worker never loads React.
    5. +
  5. Contracts carry their own versions. A plugin carries the contract version it was written for; the provider +binds it if it accepts that version, else the plugin waits with a clear reason.
    6. +
  6. A plugin imports four things: its own files, host packages, its npm dependencies, and the app's own plugins by +relative path. Nothing else.
    7. +
  7. Settings are arguments, extensions are registries. What the app decides at startup is passed to its components; +what plugins add is contributed.
    8. +
  8. No type checking in the build. Types come from where imports come from; editors and agents check, the build +only builds.
    9. +
+

Words used throughout

+

Base code is the app's own program; its API is the part plugins may import, published as API modules, +one per version and host kind. A host package is a module the running process provides to every plugin (Chord, +the app's API modules). A contract is an object type's declaration; a registry token an extension point's. +A facet is a plugin's declarations for one host kind.

+
+ +
I

What a Plugin Is

A directory or a single file, named once, with one file per host kind.

+
plugins/notes/                     a directory plugin
+  package.json                     { "name": "@acme/notes", "dependencies": { "slugify": "1.6.6" } }
+  contract.ts                      the object types and registry tokens it offers
+  worker.ts  server.ts  web.tsx    one facet per host kind it needs
+plugins/hello.worker.ts            a single-file plugin "hello" with one worker facet
+plugins/hello.web.tsx              the same plugin's web facet
+

A directory plugin's name is its package.json name; Chord reads only name and dependencies from it. A +single-file plugin's name is its file's base name; it has no dependencies and imports only host packages. When a +single file needs a package or a second file, it becomes a directory. Two roots with the same name: the first root +wins, the other is reported (slice 7c).

+
+ +
II

Base Code: Internal and External

What the app keeps to itself, what it exposes, and how exposure is versioned.

+

The app's base code is everything that starts the process: settings, startup, core objects' implementations, the +session index, migrations. Plugins must not depend on any of it, or every refactoring of the app would break them. +So the app draws a line and exposes only its API:

+
src/api/v3/index.ts       shared: host kinds, the principal type, defineFacet per kind, core contracts, registry tokens
+src/api/v3/web.ts         web only: shared UI pieces (a Sheet, theme variables), React-typed registry tokens
+src/api/v3/tui.ts         TUI only: its UI pieces
+src/…                     internal: everything else
+

Each API module is a host package of the kinds that may use it: @acme/app/v3 for every kind, @acme/app/v3/web for +the web kind only. A worker plugin importing @acme/app/v3/web is a build error, and a worker process never loads +React. The shared module stays free of runtime imports of UI libraries: a registry token that needs a React type names +it with import type, which the build erases.

+
Proposed

The split of the external API into a shared module and per-kind modules answers the owner's worry about "web, TUI, +worker, server things all in the same index.ts". Proposed, not yet decided.

+
+
+ +
III

Versions in the Import Path

A plugin says what it was written for where it imports it.

+

The app's API changes over time: a host kind is renamed, a principal variant changes. A plugin written for an older +API must not silently get the new one. So every API module has a version in its path, and the app lists which versions +it still provides:

+
// plugins/stats/worker.ts
+import { defineFacet, Session, Panels } from "@acme/app/v3";
+import { remote, str } from "@earendil-works/chord/next/v2";
+
    +
  • The app provides /v3, its current API. If version 2 plugins still work unchanged against it, /v2 is listed too, +pointing to the same module. An app that needs a compatibility layer makes /v2 a thin module over /v3.
    • +
  • An import of a version the app no longer lists is a build error: "@acme/app/v1 is not available: this app provides +v3, v2". The plugin keeps its old code, and the message tells an agent exactly what to change.
    • +
  • Nothing to fill in elsewhere: no package.json field, no npm ranges. Skeletons and examples import the current +version, so an agent copies it without thinking.
    • +
  • Chord's own surface is versioned the same way (@earendil-works/chord/next/v2).
    • +
+

Decided: "the API version must be declared by the plugin in code" (owner, 2026-10-31); the import path is how.

+
+ +
IV

Contracts and Registries

Finer versions inside the API, and what crosses between plugins.

+

Contracts and registry tokens keep their own versions inside the API, because they evolve independently of the API as +a whole:

+
    +
  • An object type: the consumer carries the contract version it was written for; the provider binds it at that +version if it accepts it (accepts maps arguments forward and results and state back), otherwise the consumer waits +"contract mismatch: v1, served v3". Shapes are fingerprinted, so a changed shape without a version bump is caught.
    • +
  • A registry token: a name, a version and TypeScript types for what is contributed and read. Values are in-process +things (functions, components), which cannot be fingerprinted: bumping the version when they change is the defining +plugin's rule, and its accepts adapts older contributions.
    • +
  • Host kinds on registries (proposed): a token names the kinds that serve it (hosts: ["web"]), typed, so a worker +facet contributing to a web registry is a type error and a load error, not a silent no-op.
    • +
+

A contract file imports only host packages at runtime (Chord, the app's API); native types it mentions are type-only.

+
+ +
V

What a Plugin May Import

Four kinds of import, each with one rule.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ImportResolved toVersioned by
its own files (./store.ts)copied into its scriptthe plugin itself
host packages (@acme/app/v3, @earendil-works/chord/next/v2, react)the running process's instance, sharedthe path's version; the app lists what it provides
npm packages in its dependencies (slugify, a third party's contracts package)its own node_modules, bundledthe installed version
the app's own plugins (../notes/contract.ts), in the same rootcopied into its scriptthe same revision: one repository
+

A third-party plugin that wants the app's plugins' contracts gets them through the app's API (core contracts) or the +app's published package, as an npm dependency at the version it was written for, never from the running plugin's +files. Anything else (a path into base code, an unlisted version, a package not in its dependencies) is a build error.

+
+ +
VI

Settings and Extensions

How the app configures its components without letting plugins override it.

+

Decided ("Settings are arguments, extensions are registries"): what the app decides at startup is an argument to the +component's facet, which base code constructs, and plugins cannot change it:

+
const http = createHttp(server, { listen: { port: 8788 }, links: { path: "/chord", maxPayload: MiB }, closeTimeout: 10_000, onError }, ctx);
+

What plugins add while the app runs is a registry: routes, authenticators, tools, panels. A component may offer both +for one thing: the session index takes its indexer as an argument, and may also follow a registry of indexers if +plugins should supply them.

+
+ +
VII

Building and Evaluating

Unchanged from slices 7c and 10: one build on the server, evaluation everywhere.

+

Only the server builds, because only the server has the sources. createPlugins builds every kind and needs only +names: per kind its platform and its host package names. Each process evaluates the scripts of its kind with its own +module objects (the server's at plugins.load, a worker's in workerMain, a page's in its base bundle), so a plugin +shares that process's Chord and API. Base-code facets are passed beside the plugins; they are not plugins, because +they close over the process's own values and never reload.

+
+ +
VIII

Types and Tests

Where types come from, and how a plugin is tested alone.

+

Types come from where imports come from: relative files and npm packages resolve natively in TypeScript; the app's API +modules ship .d.ts per version in the app's package. There is no type checking in the build (the owner: seconds per +build is too slow); editors check as an author types, and an agent may run tsc as a tool.

+

Testing a plugin alone (proposed): testPlugin(dir, { kind, stubs }) builds the plugin like production, loads it on an +in-memory host with the kind's real host packages, and loads stub facets that serve the contracts it depends on. A stub +is an ordinary facet; there is no mocking framework.

+
+ +
∴

Open Questions

What is proposed here and still to decide.

+
    +
  1. The external API split by host kind (/v3, /v3/web, /v3/tui).
    2. +
  2. Host kinds on registry tokens, typed and checked at load.
    3. +
  3. Single-file plugins (hello.worker.ts, named by the file).
    4. +
  4. Contracts of the app's plugins for third parties: through the API modules only, or also the app's npm package.
    5. +
  5. testPlugin with stub facets.
    6. +
  6. Reversing "Plugins import each other by name" (2026-10-31, morning) in favour of relative paths within the app +and npm dependencies across it.
    7. +
+
+ +

The Plugin Model · proposal · 2026-10-31

+
+ + diff --git a/packages/chord/docs/next/plugin-primer/book.md b/packages/chord/docs/next/plugin-primer/book.md new file mode 100644 index 000000000..361a043b2 --- /dev/null +++ b/packages/chord/docs/next/plugin-primer/book.md @@ -0,0 +1,273 @@ +--- +title: The Plugin Model, a Primer: what a plugin depends on, and how mismatches are caught +cover: Plugins / from the Ground Up +kicker: A primer · Chord v2 · 2026-10-31 +subtitle: One app, its own plugins and a stranger's plugin, followed through a year of changes, to show what the open decisions are about +stats: + - 4 | things a plugin depends on + - 3 | of them change without the plugin knowing + - 8 | points still to decide + - 1 | rule decided for base code: the version is in the import path +note: | + Written by the redesign session for the owner, after the proposal `docs/next/plugin-model/` did not explain enough. + Everything decided cites its decision in `decisions.md`; everything open is marked and ends with a recommendation. + Built behaviour is slice 7c and slice 10 (`src/next-node/plugins/`). +colophon: The Plugin Model, a Primer · first edition · 2026-10-31 +--- + +# Before we begin {.preface} + +Acme builds a notes app on Chord. The app is a server, a worker per project and browser tabs. Most features are +plugins: Acme's own `notes` plugin keeps the notes, its `search` plugin searches them. Bob, who does not work at +Acme, writes a `stats` plugin that counts words per project and shows a panel. Bob wrote it in March. It is now +November. + +Between March and November, three things changed without Bob: Acme released a new version of the app, Acme's `notes` +plugin changed what its notes look like, and Chord got a new release. Bob's plugin is still the March code. The +question this primer answers is the only question the plugin model has to answer: + +**When a plugin written against one state of the world runs against another, what does it get, and who notices the +mismatch?** + +The answer is different for each thing a plugin depends on, so the primer takes them one at a time. + +## What a plugin is, in one picture + +```text +plugins/stats/ Bob's plugin + package.json { "name": "@bob/stats", "dependencies": { "words-count": "2.1.0" } } + worker.ts runs in each project's worker: counts words + web.tsx runs in each tab: the panel +``` + +The server builds every plugin's files into one script per host kind (esbuild, a few milliseconds each). Each +process evaluates the scripts of its kind: the worker runs `worker.ts`'s script, a tab runs `web.tsx`'s. That much +is built (slices 7c and 10). + +# Four Things a Plugin Depends On {#four lede="Who changes each one, and when the plugin finds out."} + +Look at the top of Bob's `worker.ts`: + +```ts +import countWords from "words-count"; // 1. an npm package +import { defineFacet, Project } from "@acme/app"; // 2. the app's base code +import { obj, num, remote } from "@earendil-works/chord/next"; // 3. Chord +import { Notes } from "@acme/notes/contract"; // 4. another plugin +``` + +Each import is a different kind of dependency, and they behave differently over time: + +| Import | Who changes it | What Bob's plugin gets today | +| --- | --- | --- | +| `words-count` | its npm author | the version installed in Bob's plugin directory, copied into his script: frozen until Bob updates it | +| `@acme/app` | Acme, when it releases the app | whatever the running app is: always the newest | +| `@earendil-works/chord/next` | Chord, through Acme's upgrade | whatever the running app ships: always the newest | +| `@acme/notes/contract` | Acme, whenever it edits `notes` | the running `notes` plugin's file, copied at Bob's next build: always the newest | + +The first row is safe: npm packages are copied into the script at the version Bob installed, so they never change +under him. The other three rows always give the newest, and Bob's March code meets November's app. Whether that +works depends on what changed. The rest of the primer is about those three rows. + +:::note | Why not copy base code and Chord into the script too +Because they must be shared. If Bob's script carried its own copy of Chord, its `defineFacet` would make facets the +worker's host does not recognize, and its `Project` host kind would be a different object from the app's. Base code +and Chord are **host packages**: the process provides one instance and every plugin uses it. That is decided and +built. It is also why they can change under a plugin. +::: + +# Base Code: Versions in the Import Path {#base lede="Decided: a plugin says which app API it was written for, where it imports it."} + +In June, Acme renames the host kind `Project` to `Workspace`. Bob's March plugin imports `Project`; it no longer +exists. Without any versioning, Bob's plugin builds (the build does not type check), then fails when it runs with +something like "Cannot read properties of undefined", deep inside his code. + +Decided on 2026-10-31 (decision "API versions in the import path"): the app exposes only **API modules**, each with a +version in its path, and lists the versions it still provides. + +```ts +// Bob, in March +import { defineFacet, Project } from "@acme/app/v2"; + +// Acme's app, after June +apiVersions: { "@acme/app/v3": apiV3, "@acme/app/v2": apiV2 } // still provides v2, through a thin layer over v3 +``` + +Now there are two cases, both visible: + +- Acme still provides `/v2`: Bob's plugin gets the v2 module, in which `Project` still exists (Acme's thin layer maps + it to `Workspace`). It works. +- Acme dropped `/v2`: Bob's plugin fails to build with "@acme/app/v2 is not available: this app provides v3". The + plugin is left out (per plugin: nothing else breaks), and the message tells Bob, or an agent, exactly what to + change. + +Nothing outside the code says what Bob depends on: no `package.json` field, no version ranges. The import says it. + +**What is still open here** is what goes into the API and how Chord relates to it. That is point 2 below. + +# Other Plugins: the Contract Problem {#contracts lede="Why importing another plugin by name gives the wrong answer for a stranger's plugin."} + +Bob's `stats` asks Acme's `notes` for the notes of a project. Plugins talk only through **contracts** (the type of an +object: its state and methods) and registries. A contract has its own version: + +```ts +// plugins/notes/contract.ts (Acme), March +export const Notes = remote("notes", { + version: 1, + params: { project: str() }, + hostAddress: ({ project }) => Project({ id: project }), + variants: { reader: { state: { list: obj({ items: arr(str()) }) } } }, +}); +``` + +Bob's plugin imports this file, and the build copies it into his script. When his plugin asks for `Notes`, it says +"version 1, this shape"; the provider (the `notes` plugin's code in the worker) checks that it can serve that version. + +In September Acme changes notes: each note gets a timestamp, `items: arr(obj({ text: str(), at: int() }))`, and the +contract becomes version 2. Acme also writes an `accepts` mapping, so consumers of version 1 still get a list of +strings. Chord's contract versioning exists for exactly this case. + +Now the question is: which version of the contract file ends up in Bob's script? + +## Today: by name, the newest file + +Decided this morning and built (decision "Plugins import each other by name"): `@acme/notes/contract` resolves to the +running `notes` plugin's `contract.ts`. Trace what happens at Bob's next build after September: + +:::steps | A rebuild after Acme's change ++++ The build copies the new file +Bob's `stats` is rebuilt (a rollout, a restart, any edit anywhere). `@acme/notes/contract` is now the version 2 file, +and it is copied into Bob's script. ++++ Bob's plugin claims version 2 +At runtime his plugin asks for `Notes` version 2. The provider serves version 2 gladly: no `accepts` is used. ++++ Bob's code reads version 1's shape +Bob's code does `list.items.join(" ")` and `countWords` on it. The items are now objects. He counts the words of +`"[object Object] [object Object]"`. Nothing fails; the panel shows wrong numbers. +::: + +The contract version was supposed to protect Bob, and it did nothing: copying the newest file made Bob's plugin claim +a version his code was never written for. + +## Proposed: the version Bob wrote against + +The contract file must come from where Bob's code came from: the state of the world he wrote against. For a stranger's +plugin that is an npm dependency at a fixed version: + +```json +// plugins/stats/package.json +{ "name": "@bob/stats", "dependencies": { "@acme/notes-contracts": "1.0.0" } } +``` + +Now Bob's script carries the March contract file forever (npm packages are copied at the installed version, as row 1 +of the first table). His plugin asks for version 1; the provider maps it through `accepts`; Bob gets strings as +before. If Acme ever stops accepting version 1, Bob's plugin waits with "contract mismatch: v1, served v3", visible in +the UI, never wrong numbers. + +For Acme's own plugins (`search` using `notes`) the problem does not exist: they live in one repository and change in +the same commit. They import each other by relative path, `../notes/contract.ts`, the same file at the same revision. +The by-name rule also needed path aliases in the TypeScript configuration for every plugin (six were added for the +tests and examples of slice 10); relative paths need none. + +# The Open Points {#open lede="Eight decisions, each with the problem it answers and a recommendation."} + +## 1. Reverse "import by name" + +**Problem:** the trace above: a stranger's plugin silently claims a contract version it was not written for. +**Proposal:** the app's own plugins import each other relatively (`../notes/contract.ts`); everyone else gets +contracts as an npm dependency at a fixed version. Importing a running plugin by name is removed; a relative path +leaving the plugin's root stays a build error. +**Recommendation:** do it, in the plugin slice (slice 10 is committed with by-name imports; the change is small: +`build.ts` and the examples' imports). + +## 2. What the app's API is, and Chord in it + +**Problem:** a plugin imports two host packages that both change, `@acme/app/v3` and `@earendil-works/chord/next/v2`. +Acme must then say which combinations work: does a plugin importing app v3 and Chord v1 still run? Two version +numbers per plugin, and a matrix nobody tests. +**Proposal:** plugins import only the app's API. The app re-exports the part of Chord it supports: + +```ts +// src/api/v3/index.ts (Acme's base code) +export { obj, arr, str, int, remote, local, defineRegistry } from "@earendil-works/chord/next"; +export { Workspace, Panels, defineFacet } from "../../core.ts"; +``` + +One number, the app's, describes everything a plugin gets from its host. When Acme upgrades Chord in a way that +breaks plugins, Acme bumps its API version, and decides whether `/v2` survives as a thin layer. +**Generations:** if the app provides `/v3` and `/v2`, a facet must import from one of them only, never both (the +build checks it), because `/v2`'s `Panels` and `/v3`'s `Panels` may be different registries. +**Recommendation:** yes; app kit apps get it for free (`chord/next/app` provides the API module). + +## 3. The API split by host kind + +**Problem:** the API module holds everything plugins need, including web things: a `Sheet` component, a theme, the +`Panels` registry whose entries are React components. A worker plugin imports `@acme/app/v3` for `Workspace`, and the +worker process has to load the module, so it loads React. +**Proposal:** `/v3` holds what every host may use (contracts, registry tokens, host kinds, principals) and imports no +UI library at runtime; `/v3/web` and `/v3/tui` hold UI pieces. A worker facet importing `/v3/web` is a build error. +**Recommendation:** yes. + +## 4. Host kinds on registry tokens + +**Problem:** Bob contributes his panel from `worker.ts` by mistake (`self.contribute(Panels, …)`). The worker has no +`Panels` registry; the contribution sits in a table nobody reads. Nothing fails; the panel never appears. +**Proposal:** a token names the kinds that serve it, `defineRegistry("app.panels", { version: 1, hosts: ["web"] })`; +contributing from another kind is a type error (the facet's `self` is typed per kind) and a load error. +**Recommendation:** yes; together with point 3, `Panels` lives in `/v3/web` anyway. + +## 5. Contracts for third parties + +**Problem:** with point 1, Bob needs Acme's plugins' contracts as an npm package. Which package? +**Options:** (a) the app's API exports the contracts of its own plugins that others may use (`@acme/app/v3` has +`Notes`): one place, versioned with the app; (b) Acme publishes its app package to npm and Bob depends on it; (c) each +plugin publishes its own contracts package. (c) is what the owner called unpleasant. +**Recommendation:** (a) for contracts Acme wants others to use: the API is the app's promise. Note: in (a) the +contract file comes with the API version, so Bob's v2 import keeps v2's contract files (the thin layer keeps them). + +## 6. Testing a plugin alone + +**Problem:** testing Bob's `stats` needs a worker with a `Notes` provider, an app API, a store. +**Proposal:** `testPlugin("plugins/stats", { kind: "worker", stubs: [notesStub] })` builds the plugin as production +does, loads it on an in-memory host with the kind's real host packages, and loads stub facets that serve the +contracts it uses. A stub is an ordinary facet (`host.serve(Notes, () => ({ list: constant({ items: ["a b"] }) }))`). +**Recommendation:** yes, in the app kit. + +## 7. Single-file plugins + +**Problem:** a plugin that adds one route needs a directory and a `package.json` with a name. +**Proposal:** `plugins/hello.server.ts` is a plugin named `hello` with one server facet, no dependencies; when it +needs a package or a second file, it becomes a directory. +**Recommendation:** yes, cheap; agents write many such plugins. + +## 8. The app's own plugins follow the same rules + +**Problem:** are Acme's built-in plugins special? +**Proposal:** no. They import the app's API like everyone (not base code's internals), import each other relatively +(point 1), are built at startup and reload like any plugin. The only difference: they change in the same commit as +base code, so they always use the newest API version. +**Recommendation:** yes. It keeps base code small and means every built-in plugin is an example of a correct plugin. + +# What to Decide {.epilogue lede="In the order that makes each answer easier."} + +1. Reverse "import by name" (recommended: yes, in the plugin slice). +2. Plugins import only the app's API; Chord is re-exported in it; one generation per facet (recommended: yes). +3. Split the API by host kind (recommended: yes). +4. Host kinds on registry tokens (recommended: yes). +5. Contracts for third parties through the app's API (recommended: (a)). +6. `testPlugin` with stub facets (recommended: yes). +7. Single-file plugins (recommended: yes). +8. Built-in plugins follow the same rules (recommended: yes). + +Later, not part of these: project plugins (per working directory, with trust), and installing plugins from npm or git. + +# Words {.appendix} + +| Word | Meaning | +| --- | --- | +| base code | the app's own program, started by `node server.ts`; never reloads | +| API module | the part of base code plugins may import, one per version (`@acme/app/v3`) | +| host package | a module the process provides to every plugin (Chord, the API modules, React on the web) | +| contract | an object type: its state and methods, with a version; consumers name the version they use | +| `accepts` | the provider's mapping that serves an older contract version | +| registry token | an extension point (`Panels`); plugins contribute to it | +| facet | a plugin's code for one host kind (`worker.ts`) | +| generation | one set of API versions provided together (`/v3` with its web and tui modules) | diff --git a/packages/chord/docs/next/plugin-primer/index.html b/packages/chord/docs/next/plugin-primer/index.html new file mode 100644 index 000000000..0eb064640 --- /dev/null +++ b/packages/chord/docs/next/plugin-primer/index.html @@ -0,0 +1,430 @@ + + +The Plugin Model, a Primer: what a plugin depends on, and how mismatches are caught + + + + +
+
+
A primer · Chord v2 · 2026-10-31
+

Plugins
from the Ground Up

+

One app, its own plugins and a stranger's plugin, followed through a year of changes, to show what the open decisions are about

+
4things a plugin depends on
3of them change without the plugin knowing
8points still to decide
1rule decided for base code: the version is in the import path
+

Written by the redesign session for the owner, after the proposal docs/next/plugin-model/ did not explain enough. +Everything decided cites its decision in decisions.md; everything open is marked and ends with a recommendation. +Built behaviour is slice 7c and slice 10 (src/next-node/plugins/).

+
+
+ + +
+

Before we begin

+

Acme builds a notes app on Chord. The app is a server, a worker per project and browser tabs. Most features are +plugins: Acme's own notes plugin keeps the notes, its search plugin searches them. Bob, who does not work at +Acme, writes a stats plugin that counts words per project and shows a panel. Bob wrote it in March. It is now +November.

+

Between March and November, three things changed without Bob: Acme released a new version of the app, Acme's notes +plugin changed what its notes look like, and Chord got a new release. Bob's plugin is still the March code. The +question this primer answers is the only question the plugin model has to answer:

+

When a plugin written against one state of the world runs against another, what does it get, and who notices the +mismatch?

+

The answer is different for each thing a plugin depends on, so the primer takes them one at a time.

+

What a plugin is, in one picture

+
plugins/stats/                     Bob's plugin
+  package.json                     { "name": "@bob/stats", "dependencies": { "words-count": "2.1.0" } }
+  worker.ts                        runs in each project's worker: counts words
+  web.tsx                          runs in each tab: the panel
+

The server builds every plugin's files into one script per host kind (esbuild, a few milliseconds each). Each +process evaluates the scripts of its kind: the worker runs worker.ts's script, a tab runs web.tsx's. That much +is built (slices 7c and 10).

+
+ +
I

Four Things a Plugin Depends On

Who changes each one, and when the plugin finds out.

+

Look at the top of Bob's worker.ts:

+
import countWords from "words-count";                       // 1. an npm package
+import { defineFacet, Project } from "@acme/app";           // 2. the app's base code
+import { obj, num, remote } from "@earendil-works/chord/next";   // 3. Chord
+import { Notes } from "@acme/notes/contract";               // 4. another plugin
+

Each import is a different kind of dependency, and they behave differently over time:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ImportWho changes itWhat Bob's plugin gets today
words-countits npm authorthe version installed in Bob's plugin directory, copied into his script: frozen until Bob updates it
@acme/appAcme, when it releases the appwhatever the running app is: always the newest
@earendil-works/chord/nextChord, through Acme's upgradewhatever the running app ships: always the newest
@acme/notes/contractAcme, whenever it edits notesthe running notes plugin's file, copied at Bob's next build: always the newest
+

The first row is safe: npm packages are copied into the script at the version Bob installed, so they never change +under him. The other three rows always give the newest, and Bob's March code meets November's app. Whether that +works depends on what changed. The rest of the primer is about those three rows.

+
Why not copy base code and Chord into the script too

Because they must be shared. If Bob's script carried its own copy of Chord, its defineFacet would make facets the +worker's host does not recognize, and its Project host kind would be a different object from the app's. Base code +and Chord are host packages: the process provides one instance and every plugin uses it. That is decided and +built. It is also why they can change under a plugin.

+
+
+ +
II

Base Code: Versions in the Import Path

Decided: a plugin says which app API it was written for, where it imports it.

+

In June, Acme renames the host kind Project to Workspace. Bob's March plugin imports Project; it no longer +exists. Without any versioning, Bob's plugin builds (the build does not type check), then fails when it runs with +something like "Cannot read properties of undefined", deep inside his code.

+

Decided on 2026-10-31 (decision "API versions in the import path"): the app exposes only API modules, each with a +version in its path, and lists the versions it still provides.

+
// Bob, in March
+import { defineFacet, Project } from "@acme/app/v2";
+
+// Acme's app, after June
+apiVersions: { "@acme/app/v3": apiV3, "@acme/app/v2": apiV2 }   // still provides v2, through a thin layer over v3
+

Now there are two cases, both visible:

+
    +
  • Acme still provides /v2: Bob's plugin gets the v2 module, in which Project still exists (Acme's thin layer maps +it to Workspace). It works.
    • +
  • Acme dropped /v2: Bob's plugin fails to build with "@acme/app/v2 is not available: this app provides v3". The +plugin is left out (per plugin: nothing else breaks), and the message tells Bob, or an agent, exactly what to +change.
    • +
+

Nothing outside the code says what Bob depends on: no package.json field, no version ranges. The import says it.

+

What is still open here is what goes into the API and how Chord relates to it. That is point 2 below.

+
+ +
III

Other Plugins: the Contract Problem

Why importing another plugin by name gives the wrong answer for a stranger's plugin.

+

Bob's stats asks Acme's notes for the notes of a project. Plugins talk only through contracts (the type of an +object: its state and methods) and registries. A contract has its own version:

+
// plugins/notes/contract.ts (Acme), March
+export const Notes = remote("notes", {
+	version: 1,
+	params: { project: str() },
+	hostAddress: ({ project }) => Project({ id: project }),
+	variants: { reader: { state: { list: obj({ items: arr(str()) }) } } },
+});
+

Bob's plugin imports this file, and the build copies it into his script. When his plugin asks for Notes, it says +"version 1, this shape"; the provider (the notes plugin's code in the worker) checks that it can serve that version.

+

In September Acme changes notes: each note gets a timestamp, items: arr(obj({ text: str(), at: int() })), and the +contract becomes version 2. Acme also writes an accepts mapping, so consumers of version 1 still get a list of +strings. Chord's contract versioning exists for exactly this case.

+

Now the question is: which version of the contract file ends up in Bob's script?

+

Today: by name, the newest file

+

Decided this morning and built (decision "Plugins import each other by name"): @acme/notes/contract resolves to the +running notes plugin's contract.ts. Trace what happens at Bob's next build after September:

+
A rebuild after Acme's change

The build copies the new file

Bob's stats is rebuilt (a rollout, a restart, any edit anywhere). @acme/notes/contract is now the version 2 file, +and it is copied into Bob's script.

+

Bob's plugin claims version 2

At runtime his plugin asks for Notes version 2. The provider serves version 2 gladly: no accepts is used.

+

Bob's code reads version 1's shape

Bob's code does list.items.join(" ") and countWords on it. The items are now objects. He counts the words of +"[object Object] [object Object]". Nothing fails; the panel shows wrong numbers.

+
+

The contract version was supposed to protect Bob, and it did nothing: copying the newest file made Bob's plugin claim +a version his code was never written for.

+

Proposed: the version Bob wrote against

+

The contract file must come from where Bob's code came from: the state of the world he wrote against. For a stranger's +plugin that is an npm dependency at a fixed version:

+
// plugins/stats/package.json
+{ "name": "@bob/stats", "dependencies": { "@acme/notes-contracts": "1.0.0" } }
+

Now Bob's script carries the March contract file forever (npm packages are copied at the installed version, as row 1 +of the first table). His plugin asks for version 1; the provider maps it through accepts; Bob gets strings as +before. If Acme ever stops accepting version 1, Bob's plugin waits with "contract mismatch: v1, served v3", visible in +the UI, never wrong numbers.

+

For Acme's own plugins (search using notes) the problem does not exist: they live in one repository and change in +the same commit. They import each other by relative path, ../notes/contract.ts, the same file at the same revision. +The by-name rule also needed path aliases in the TypeScript configuration for every plugin (six were added for the +tests and examples of slice 10); relative paths need none.

+
+ +
IV

The Open Points

Eight decisions, each with the problem it answers and a recommendation.

+

1. Reverse "import by name"

+

Problem: the trace above: a stranger's plugin silently claims a contract version it was not written for. +Proposal: the app's own plugins import each other relatively (../notes/contract.ts); everyone else gets +contracts as an npm dependency at a fixed version. Importing a running plugin by name is removed; a relative path +leaving the plugin's root stays a build error. +Recommendation: do it, in the plugin slice (slice 10 is committed with by-name imports; the change is small: +build.ts and the examples' imports).

+

2. What the app's API is, and Chord in it

+

Problem: a plugin imports two host packages that both change, @acme/app/v3 and @earendil-works/chord/next/v2. +Acme must then say which combinations work: does a plugin importing app v3 and Chord v1 still run? Two version +numbers per plugin, and a matrix nobody tests. +Proposal: plugins import only the app's API. The app re-exports the part of Chord it supports:

+
// src/api/v3/index.ts (Acme's base code)
+export { obj, arr, str, int, remote, local, defineRegistry } from "@earendil-works/chord/next";
+export { Workspace, Panels, defineFacet } from "../../core.ts";
+

One number, the app's, describes everything a plugin gets from its host. When Acme upgrades Chord in a way that +breaks plugins, Acme bumps its API version, and decides whether /v2 survives as a thin layer. +Generations: if the app provides /v3 and /v2, a facet must import from one of them only, never both (the +build checks it), because /v2's Panels and /v3's Panels may be different registries. +Recommendation: yes; app kit apps get it for free (chord/next/app provides the API module).

+

3. The API split by host kind

+

Problem: the API module holds everything plugins need, including web things: a Sheet component, a theme, the +Panels registry whose entries are React components. A worker plugin imports @acme/app/v3 for Workspace, and the +worker process has to load the module, so it loads React. +Proposal: /v3 holds what every host may use (contracts, registry tokens, host kinds, principals) and imports no +UI library at runtime; /v3/web and /v3/tui hold UI pieces. A worker facet importing /v3/web is a build error. +Recommendation: yes.

+

4. Host kinds on registry tokens

+

Problem: Bob contributes his panel from worker.ts by mistake (self.contribute(Panels, …)). The worker has no +Panels registry; the contribution sits in a table nobody reads. Nothing fails; the panel never appears. +Proposal: a token names the kinds that serve it, defineRegistry("app.panels", { version: 1, hosts: ["web"] }); +contributing from another kind is a type error (the facet's self is typed per kind) and a load error. +Recommendation: yes; together with point 3, Panels lives in /v3/web anyway.

+

5. Contracts for third parties

+

Problem: with point 1, Bob needs Acme's plugins' contracts as an npm package. Which package? +Options: (a) the app's API exports the contracts of its own plugins that others may use (@acme/app/v3 has +Notes): one place, versioned with the app; (b) Acme publishes its app package to npm and Bob depends on it; (c) each +plugin publishes its own contracts package. (c) is what the owner called unpleasant. +Recommendation: (a) for contracts Acme wants others to use: the API is the app's promise. Note: in (a) the +contract file comes with the API version, so Bob's v2 import keeps v2's contract files (the thin layer keeps them).

+

6. Testing a plugin alone

+

Problem: testing Bob's stats needs a worker with a Notes provider, an app API, a store. +Proposal: testPlugin("plugins/stats", { kind: "worker", stubs: [notesStub] }) builds the plugin as production +does, loads it on an in-memory host with the kind's real host packages, and loads stub facets that serve the +contracts it uses. A stub is an ordinary facet (host.serve(Notes, () => ({ list: constant({ items: ["a b"] }) }))). +Recommendation: yes, in the app kit.

+

7. Single-file plugins

+

Problem: a plugin that adds one route needs a directory and a package.json with a name. +Proposal: plugins/hello.server.ts is a plugin named hello with one server facet, no dependencies; when it +needs a package or a second file, it becomes a directory. +Recommendation: yes, cheap; agents write many such plugins.

+

8. The app's own plugins follow the same rules

+

Problem: are Acme's built-in plugins special? +Proposal: no. They import the app's API like everyone (not base code's internals), import each other relatively +(point 1), are built at startup and reload like any plugin. The only difference: they change in the same commit as +base code, so they always use the newest API version. +Recommendation: yes. It keeps base code small and means every built-in plugin is an example of a correct plugin.

+
+ +
∴

What to Decide

In the order that makes each answer easier.

+
    +
  1. Reverse "import by name" (recommended: yes, in the plugin slice).
    2. +
  2. Plugins import only the app's API; Chord is re-exported in it; one generation per facet (recommended: yes).
    3. +
  3. Split the API by host kind (recommended: yes).
    4. +
  4. Host kinds on registry tokens (recommended: yes).
    5. +
  5. Contracts for third parties through the app's API (recommended: (a)).
    6. +
  6. testPlugin with stub facets (recommended: yes).
    7. +
  7. Single-file plugins (recommended: yes).
    8. +
  8. Built-in plugins follow the same rules (recommended: yes).
    9. +
+

Later, not part of these: project plugins (per working directory, with trust), and installing plugins from npm or git.

+
+ +
A

Words

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
WordMeaning
base codethe app's own program, started by node server.ts; never reloads
API modulethe part of base code plugins may import, one per version (@acme/app/v3)
host packagea module the process provides to every plugin (Chord, the API modules, React on the web)
contractan object type: its state and methods, with a version; consumers name the version they use
acceptsthe provider's mapping that serves an older contract version
registry tokenan extension point (Panels); plugins contribute to it
faceta plugin's code for one host kind (worker.ts)
generationone set of API versions provided together (/v3 with its web and tui modules)
+
+ +

The Plugin Model, a Primer · first edition · 2026-10-31

+
+ + diff --git a/packages/chord/docs/next/slice-10.md b/packages/chord/docs/next/slice-10.md new file mode 100644 index 000000000..6bbaea50a --- /dev/null +++ b/packages/chord/docs/next/slice-10.md @@ -0,0 +1,285 @@ +# Slice 10: HTTP and workers (spec) + +Status: **built** (2026-10-31; build reviews by Fable, Opus, Astra and Sol, security audits by Fable, Opus and Astra, four fix rounds with rechecks; the shape after the fix rounds is in §2, §3 and design §23). Approved by the owner (2026-10-30: "just build it"; D1-D12, D10b, round 1's R1-R6 and round 2's Q1-Q6 in the +decision "Slice 10's points"; spec reviews round 1 by Fable and Opus, `/tmp/s10-spec-review-{fable,opus}.md`, round 2 +by Fable, Opus, Astra and Sol, `/tmp/s10-spec-review2-*.md`, folded in). +Slice 10 builds two Node components of the app kit (decision "Slice 7b's open points" (6)): `chord/next/http` (the +server's HTTP and WebSocket entry, routes contributed by plugins through a registry, responses as descriptions) and +`chord/next/workers` (a worker process per host address, started on demand, stopped when idle, its plugins shipped from +the server). Both live in `packages/chord/src/next-node/` (Node-only); the kernel stays portable. The web page host and +the TUI are slice 11, telemetry slice 9. + +Read in full before writing code: this spec; `decisions.md`; `design.md` (§3 hosts, routes, `host.activity`, §17, +§18, §21 "Loading plugins and the reload loop", §23 in full); `slices.md`; `slice-7c.md` and +`src/next-node/plugins/README.md`; the 7b brief `slice-7b/book.md` (chapters "Assets and Files", "HTTP", "A Mini p2 +from Chord"); p2's `src/hosts/server.ts`, `src/runtime/server.ts`, `src/hosts/worker.ts`, `src/hosts/socket-limits.ts`, +`src/hosts/principals.ts`, `plugins/files/server.ts`, `plugins/terminals/server.ts`, `plugins/radius/server.ts`; Chord +v1's `src/manager.ts` (`processLock`, `forkDriver`, `createHostManager`, `childControl`). + +## 1. What p2 has, and what moves into Chord + +p2's server host (`src/hosts/server.ts`, 742 lines; `src/runtime/server.ts`, 720) does, besides p2's own policy: + +| p2 today | Moves to | +| --- | --- | +| a Node `http` server on loopback, requests parked until `load` resolved, upgrades captured before listening | `http` | +| `Routes` (`"METHOD /path"`, `/prefix/*`), handlers writing Node's `ServerResponse` themselves (ranges, download headers, `HEAD` by hand in `files`) | `http`: routes returning descriptions | +| `Upgrades` (a WebSocket by path for terminals and voice), `/chord` (`acceptWebSockets` with per-socket limits) | `http` | +| `Authenticators`, `RequestResolvers` (cookies, devices, the Radius relay), Fetch Metadata, CSP and the loopback cookie | the app (p3's base code), through `http`'s `authenticate` and `headers` (D3) | +| loopback and allowed hosts, the `Origin` check on upgrades | `http` (R4), allowed hosts contributable by plugins | +| `SessionHosts`: a worker per session, `createHostManager` + `forkDriver`, a kernel lock per session, idle stop from reported counts, crash notices, sticky failed starts, ready timeout | `workers` (failures stay until Retry or a rollout, R2) | +| relocation and pause of a worker, the catalog, `WorkerInitValues` | the app (built on `workers`' `stop` and start data) | +| `hosts/worker.ts`: the child's control channel, lock, start data, connect, load, report counts, shutdown | `workers`' child side | +| the dev reload across server and workers, worker plugin lists | `plugins` (7c) plus `workers`' rollout step (§4) | + +Every option that is not bootstrap is changeable by plugins (decision "The app kit is malleable by plugins"): either a +contribution to `Http` or a function the component calls per use, which the app backs with its own registries; +nothing is copied out of a contribution or an option at startup. Bootstrap (changed by a restart): the listen address +and port, the links path, the worker entry with its `execArgv` and `env`, the host kind workers serve. + +## 2. `chord/next/http` + +As built after the fix rounds (decisions "The HTTP core is a fetch handler", "The HTTP server is a native object", +"Settings are arguments, extensions are registries", "The HTTP component's security baseline", "allowed hosts only +through contributions", "A CSP nonce per response", "three small points", "a route's overhead accepted"); design §23 +states every rule, `src/next-node/http/README.md` the API. + +```ts +const http = createHttp(server, { + listen: { host: "127.0.0.1", port: 8788 }, // settings: arguments, a restart to change + links: { path: "/chord", options: (request, principal) => ({ limits: linkLimits(principal), maxPayload: 64 * MiB }) }, + closeTimeout: 30_000, + onError: (error) => log(error), // `HttpError`, below +}, ctx); +const app = defineFacet((_host, self) => { + self.contribute(Authenticators, (request, ctx) => principalFromCookie(readCookie(request, "session", secret))); + self.contribute(Http, { hosts: ["notes.example.com"], relays: [radiusRelay] }); +}); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }, { name: "@app/core", facet: app }], modules }, ctx); +… +await http.close(ctx); // first: no new requests, sockets closed, running requests waited for up to `closeTimeout` +await workers.close(ctx); +await server.close(ctx); +``` + +- **The core** is `http.fetch(request, { localAddress?, localPort?, remoteAddress? })`, a Web `Request` to a Web + `Response`; `http.upgrade` is Node's upgrade adapter; `nodeRequest(message, res)` converts a Node request with the + listener's checks (`{ kind: "request", request, connection } | { kind: "refused", status }`, never a throw) for apps + mounting on their own Node server. Chord's own listener (with `listen`) is a host-lifetime object of `chord.http` + whose native is a Node server, on a fast path that skips the Web `Request` unless a route or authenticator reads + it. +- **Registries:** `Http` (`"chord.http"`, version 1: `routes`, `wraps`, `sockets`, `maxPayload`, `hosts`, `framers`, + `relays`, `csp`; a value or a writable) and `Authenticators` (in contributor order, the first principal wins). + Requests before the publication wait for it (503 past `readyTimeout`). +- **Admission, foreign requests, origins, default headers, cookies, downloads:** the security baseline (design §23). + Refusals before any plugin code: a non-loopback socket without an allowed host, forwarded headers without a + verified relay, duplicate or missing `Host`, userinfo in `Host`, absolute-form targets (403), unparsable targets + (400), `TRACE`/`TRACK`/`CONNECT` (501; a route for them refused at compose). +- **A request's scope, `HttpRequest`, patterns, wraps, responses, `fileResponse`, sockets, links, revocation, + `close`:** design §23. The spec's earlier text (an `httpServer` that listens first, `allowedHosts`, one + `authenticate` and a `headers` option, 64 KiB reads) is superseded by those decisions. +- **Errors** (`onError`, `HttpError`): `route_failed { pattern, plugin, error }`, `wrap_failed { name, plugin, error }`, + `socket_failed { pattern, plugin, error }`, `response_failed { pattern, error }`, `hook_failed { hook, plugin?, + error }` (`authenticate`, `relay`, `links.options`). +- `ws` is an optional peer dependency (D7), imported only here. + +## 3. `chord/next/workers` + +```ts +// server.ts +const workers = createWorkers(server, Session, { + kind: "worker", // the facet kind it ships + entry: "./worker.ts", execArgv: ["--import", resolver], env: { … }, // bootstrap + authorize: (principal, address) => app.mayReach(principal, address), // the route's `authorize` (synchronous) + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: (address, ctx) => app.startData(address, ctx), // JSON, before each start + lock: (address) => `app:${realpath(dataDir)}:worker:${address.key}`, // one live worker per address + idleMs: () => app.idleMs(), + plugins, // the plugins component + onOutput: (address, stream, chunk) => logs.append(address.key, stream, chunk), + onError: (error) => log(error), // `WorkersError`, below +}, ctx); +workers.state // JSON watchable (R5) +await workers.stop(Session({ id }), ctx); // archive: suspended while held (Q3), then stopped +await workers.retry(Session({ id }), ctx); // the Retry button: a failed or suspended address resolves again +await workers.close(ctx); // no new starts, every child stopped +``` + +```ts +// worker.ts +await workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(start.dir, "session.sqlite"))), ctx); + onClose((ctx) => store.close(ctx)); // closers run in reverse order + const worker = createHost

({ address, principal: { kind: "worker", session: start.id }, store, onError }); + onClose((ctx) => worker.close(ctx)); + worker.connect(parent, { principal: { kind: "server" }, trusted: true }, ctx); // `started` goes out now: connect + // before anything slow (a store opens in ms) + await plugins.load(worker, { kind: "worker", core: [{ name: "@app/core", facet: coreWorker(start) }] }, ctx); + return { busy: harness.activeTasks }; // extra keep-alive counts +}, { + hostPackages: { "@earendil-works/chord/next": chordNext, "@app/core": appCore }, // the child's own instances + redact: (error) => ({ message: redactSecrets(String(error)), ...codeOf(error) }), // what crosses as `fatal` + onError: (error) => log(error), +}); +``` + +- **The route:** `createWorkers` calls `host.route(Kind, { authorize, resolve }, ctx)` itself and keeps the + `RouteHandle` (`retry`, `suspend`: D10b, R1). `authorize` is synchronous (the kernel's): a plugin backing it keeps an + index. +- **Per address, events and operations:** a child's events (its exit, its link's close, `started`, `ready`, `fatal`) + are observed as they happen and recorded at once on that child (a late event of an old child never changes a new + one); the operations (start, stop) run in one queue per address: a start waits for the previous child's exit, a + stop during a start waits for it then stops, and a start queued behind an exit checks the address's state after it + (a crash recorded meanwhile makes it reject with that failure). Stops record their intent before `shutdown`, so the + exit they cause is no crash. +- **A start** (`resolve(address)`): calls `start(address, ctx)` (abortable; a throw rejects the start), checks that the + component is not closing, forks `entry` (`serialization: "advanced"`, a detached process group, stdout and stderr + piped: the stderr tail kept, mirrored to the server's when there is no `onOutput`), makes `ipcLink(child)` at once + (so the child's `hello` and early holds are queued, never lost), and sends the start message `{ address, start, + scripts, lock }` (`scripts`: the worker kind's complete current set, read when the message is sent). The child takes + the lock (when an old worker still holds it, it waits by connecting to the lock's socket and trying again when that + connection closes: no polling), and sends `started` when the app's host sets its listener on `parent` (`parent` is + a thin wrapper of the IPC link, so `connect` is the signal: the app connects before anything slow). `resolve` then + returns `{ link, principal: principal(address), trusted: true }`, and the handshake and the worker's server traffic + run during its `load`. The start timer (60 s from the fork) covers fork, lock and connect only (Q4): past it the tree is + killed and the start fails. `ready` (after the app's function returned, carrying the idle state and its load report) sets `state` to + `running` and lets idle stops begin; a `load` that never ends shows `starting` until `stop`. +- **Workers never restart by themselves** (R2): a crash (an exit nobody asked for), a `fatal`, a link lost while its + child lives, or a failed start (`start` threw, the start timer, a lock not taken in time) leaves the address `failed` + with its error (`worker_failed` reported once). A link lost while its child lives (the heartbeat after a blocked + event loop, a protocol error) starts the stop: the grace (5 s), then the tree is killed; its exit is the crash ("link + lost"). `resolve` of a failed address rejects at once with that error; its holders and new holds see `failed { + waiting }` with it as cause (Q1) and stay on the address's path, nothing forks. `workers.retry(address)` + (`RouteHandle.retry`) or a rollout reaching workers clears it (Q2, §4). An idle stop is no failure: the next hold + starts the worker on demand. +- **Keep-alive** (D9, R3, Q5): the child's idle state is all of `host.activity`'s `references`, `jobs` and `lingering` + and the app's `busy` (a JSON watchable of numbers) at zero, not `calls` (a remote call runs on a hold; local calls + come from routines, jobs and tasks counted elsewhere). `ready { idle }` carries the first state, then `{ idle }` on + each flip; a rollout applying in the child counts as busy. The server stops a worker idle since `idleMs()` (5 min by + default; checked every second), never before `ready`. A stop (idle, `stop`, `close`): `shutdown`; the child aborts its + `ctx`, runs the closers registered with `onClose` in reverse order and exits, forcing its own exit after 3 s; the + server kills the tree after 5 s. +- **`state`** (R5, decision "worker errors in public"): per address key `{ status: "starting" | "running" | + "stopping", pid, startedAt }` or `{ status: "failed", error: { message, code?, reason? }, at }` with a public error + (a fixed message, or the child's redacted `fatal`); no entry when stopped or never started. The full error and the + stderr tail go only to `onError` (`worker_failed`); every app hook of a start (`start`, `principal`, `lock`, the + fork) is inside that boundary. +- **`stop(address, ctx)`** (R1, Q3): `RouteHandle.suspend(address)`, then the stop. The suspension lasts while the + address has holders (they see `failed { waiting }`, "suspended"); with the last holder it goes, and a later hold + starts the worker on demand; `retry(address)` lifts it at once. Whether an archived session may be opened is the + app's (`authorize`, its views). +- **`close(ctx)`:** cancels active and queued starts first (a `start(address)` waiting on the app's barrier is + aborted; nothing forks after it), then stops every child (grace, then kill), clears the timers; new starts reject + `released/closed`; the caller's cancellation ends only its own wait. `createWorkers`'s `ctx` aborting (the host + closing aborts the route's context) closes it too, and from then on every exit is a stop, never a crash (no + `worker_failed` at shutdown). +- **The child** (`workerMain(fn, { hostPackages, redact, onError })`): reads the start message, takes the lock, makes + `ipcLink(process)` and its wrapper `parent`, creates its plugin runtime with `hostPackages` (the child's own module + instances; functions cannot cross IPC), calls `fn({ address, start, parent, plugins, onClose }, ctx)`. `shutdown`, + the parent's disconnect, `SIGTERM` and `SIGINT` alike abort `ctx` and run the closers registered so far, also while + `fn` still runs (its startup is cancelled through `ctx`). An uncaught error, or `fn` throwing, sends `fatal` with + `redact(error)` (never the original message) and exits; after `started` that is a crash like any other. +- **Control messages** (JSON beside the link's `Uint8Array` frames; the IPC link skips them): parent to child `start`, + `shutdown`, `reload { scripts }`; child to parent `started`, `ready { idle }`, `idle { idle }`, `fatal { error }`, + `reloaded { report } | { error }`. One `reload` is outstanding per child at a time: a reload whose caller aborted + keeps the child's slot until its `reloaded` or the child's exit, and the next waits for it. Errors cross as `{ + message, code?, reason? }`. +- **Errors** (`onError`, `WorkersError`, which includes `BatchError` for a `state` watch listener that throws): `worker_failed { address, error, stderrTail? }` (private), + `authorize_failed { address, error }` (a throwing `authorize`: the hold is refused `forbidden`), `worker_exited { + address, code, signal }` (only for an exit nobody asked for: not after a stop or a start the component failed). + +### Plugins in workers (D11) + +Workers never build. The plugins component (7c) gets `scripts(kind, ctx)`: the complete current set of one kind (every +discovered plugin's facet of it, built on first use, then updated by every rollout that ships that kind), each with its +plugin record (`name`, `dir`, `chord.external`). The component's per-host loading (its loaded map, `#prepare`, `adopt`, +the queue per host) moves into a file without esbuild or the watcher, `createPluginRuntime()` +with `load(host, { kind, core, scripts, modules }, ctx)` (`modules`: the host packages' module objects of this process) and `reload(host, scripts, ctx)` (a complete set: removed = loaded − +shipped; same digests unchanged); `createPlugins` uses it, and the child's `plugins` is a `WorkerPlugins` over one +(`load(host, { kind, core })` loads the start message's scripts; `reload` messages wait until `fn` returned); `import` +of `chord/next/plugins/runtime` loads no esbuild. Each kind is defined once in `createPlugins({ kinds: { : { +platform, hostPackages } } })` (decision "Slice 11's points", Kinds). + +## 4. The rollout across server and workers + +```ts +const built = await plugins.build(plugins.state.value.dirty, ctx); +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx); +// result.steps: [{ name: "server", hosts: [{ host: "server", report }] }, +// { name: "workers", hosts: [{ host: "session|S1", report }, { host: "session|S2", stopped: true }, +// { host: "session|S3", error }] }] +``` + +`RolloutStep.reload(built, ctx)` returns `{ hosts: readonly ({ host; report } | { host; stopped: true } | { host; error +})[] }` (D12; 7c's in-process step returns a list of one, its `host` the name given to `step`). `workers.step(name)`: + +1. replaces the workers' current set (`plugins.scripts(kind)` updated from `built`); +2. clears the failures recorded before this rollout and calls `RouteHandle.retry` for those addresses that have holders + (Q2): they start with the new set and are listed; an unheld one starts at its next hold; +3. ships the complete set to every running or starting worker whose start message or last reload carried an older one + (a child queues a reload that arrives before its `ready` until its load is done); workers reload in parallel. + +A worker's entry settles on its `reloaded` or on its exit or link loss, whichever comes first. A worker stopped cleanly +during the step (idle, `stop`) is listed `stopped` and dirties nothing (its next start gets the current set); one that +crashed or failed to apply is listed with its error and does not stop the rollout, and every plugin of its kind in the +build counts as failed there (stays dirty); a worker failing during this rollout stays failed. A plugin's dirty mark +clears only when it applied on every host of every step, and a kind built in the rollout that no step shipped keeps its +plugins dirty. + +## 5. Not in slice 10 + +The web page host, the browser's evaluation of scripts and serving them (slice 11); the TUI (slice 11); relocation and +pause of workers (the app's: `start(address)` is where the app awaits its own barrier); request resolvers and +authenticators as registries (the app's); chunked transfers as a Chord helper (an owner pattern, design §23); the +channel component (D6); telemetry spans for requests (slice 9). + +## 6. Kernel changes + +- `host.route(Kind, options, ctx)` returns a `RouteHandle { retry(address), suspend(address) }` (D10b, R1, Q3). +- A route has no backoff timer (R2): a rejected `resolve`, or a link that closes before its handshake completed, fails + the route's holders `failed { waiting }` with its error (they stay on the address's path, Q1) and resolves nothing + more; the route resolves again only for a new hold, on `retry`, or once after a link that had completed its handshake + dropped while holds exist; a suspended address resolves nothing while it has holders and forgets the suspension with + the last one. +- `host.root(fn, ctx, { principal: undefined })` runs anonymous (`ctx.principal` undefined); only an omitted + `principal` takes the host's (a public-API change of `RootOptions`: `"principal" in options`). +- Origins only from relays (decision, 2026-10-31, security audit): `origin` is read only on parent and route links; + on a child link it is a protocol error; a per-origin record goes once it has no holds, calls or messages in flight. +- An anonymous `host.root` scope gets no remote object served on its host (`forbidden`). +- Design §3: the paragraph on a `resolve` waiting for the worker's `load` is updated (done), and "Chord never closes a + route link itself except at `host.close`" becomes "except at `host.close`, the heartbeat and protocol errors". + +## 7. Tests, examples, cost + +- One test file per source file. HTTP: listening before the publication (a request waits, then is served; 503 past + `readyTimeout`); the router resolved once; `Host` (loopback forms, allowed from the option and from a writable + contribution changing at runtime) and `Origin` checks; anonymous requests (`ctx.principal` undefined on a server with + a principal); authentication per request and upgrade; throwing hooks; patterns, decoding, precedence, `HEAD`, 404; + conflicts reported; wraps (phases, order, scopes, a missing route, a throwing wrap); every response kind (ranges + 206/416, multi-range, `HEAD`, download names, chunked without `size`, a short `read`, a disconnect stopping `read`, + header merging); `json`/`body` limits and their statuses, an unread body; a throwing route; the links endpoint + (`host.accept`, options per socket, a refused socket, the output cap); plugin sockets (an oversized frame closes only + that socket; a reload removing the contribution closes its sockets 1012); a removed allowed host closing its + connections; a reload swapping a route while a request runs on the old router; `fileResponse` closing its handle on + every path; `close` (refusals, waiting, then sealing). Workers: a start on the first hold; the link made at the fork + (a load longer than the heartbeat's 30 s binds); `started` on `connect`; the start timer covering only the start (a + slow `load` is not killed); the lock (an old worker's lock waited for without polling); `start` throwing; a crash, + a `fatal`, a link lost with a live child (killed after the grace), and a failed start each staying failed (no fork for + a new hold; holders `failed { waiting }`), `retry` and a rollout clearing them and rebinding the holders; idle stop, + a worker idle from its start, a hold keeping it; `busy`; 1,000 sequential local calls sending no message; + `stop` with holders (suspended, then forgotten with the last holder); stop during start and start during stop; a + crash while a resolve is queued; `close` (cancelling a start waiting in `start(address)`, no `worker_failed` at + shutdown); the rollout step (complete sets, a worker starting during it, a clean stop listed `stopped`, a crash during + it, an aborted reload's late reply, a kind no step shipped staying dirty); the child's shutdown during its startup, + disconnect and signals; `redact` (a secret in the original message reaches neither `fatal` nor `state`). Kernel: + `RouteHandle.retry` and `suspend`; a link closing before its handshake (no re-resolve loop); no re-resolve without a + hold, a retry or one drop of a healthy link; `host.root` with an explicit `undefined` principal. +- GC: requests (their scopes, bodies, file handles) and stopped workers' state collected; no listener left on a closed + socket; 1,000 addresses started and stopped leave no `state` entries, no per-address queue records and no + suspensions. +- Examples (each a folder, as 7c's): `40-http` (routes, a wrap, a download with ranges, a socket, an allowed host + added at runtime), `41-workers` (a server routing a session kind to forked workers, idle stop, a crash staying failed + until Retry, a rollout reaching the workers and restarting a held failed one). +- Benchmark: a route's overhead per request against a plain Node handler (bar: within 10% of a keep-alive round + trip; Opus measured 2.5 µs of 54 µs); `bytes` throughput for a 100 MiB file against `pipeline`; a worker's cold start + (fork to `started` and to `ready`) with shipped scripts; memory per idle worker on the server. diff --git a/packages/chord/docs/next/slice-11.md b/packages/chord/docs/next/slice-11.md new file mode 100644 index 000000000..43403bd20 --- /dev/null +++ b/packages/chord/docs/next/slice-11.md @@ -0,0 +1,114 @@ +# Slice 11: web and TUI surfaces (spec) + +Status: **approved** (owner, 2026-10-30/11-01: P1-P9 in the decisions "Slice 11's points" and "Slice 11, P3" to "P9", +then "go ahead, just do all the things"). Built after slices 12 and 13: the surfaces are generic plumbing that the app +kit wires in (`startApp` serves the web surface; a TUI client entry). The app keeps everything visible: HTML shell, +CSS, components, themes, navigation, layout, keys. + +Read in full before writing code: this spec; `decisions.md` (slice 11's entries, the plugin model, the app kit); +`design.md` §3 (clients), §21, §23 (all, "Web and TUI" included); `slice-10.md`, `slice-12.md`, `slice-13.md`; p2's +`src/hosts/web.ts`, `src/hosts/web/main.tsx`, `src/web/reconnect.ts`, `src/runtime/web.tsx`, `src/hosts/bundles.ts`, +`src/hosts/tui.ts`, `src/hosts/tui-client.ts`, `src/hosts/tui-sandbox.ts`, `src/hosts/tui-bootstrap.mjs`, +`src/hosts/tui-worker-runtime.ts`, `src/runtime/tui.ts` (`TuiScope`). + +## 1. `chord/next/web` (server side) + +```ts +// inside startApp (slice 13), or by hand: +const web = webSurface(server, { + kind: "web", plugins, + entry: "./src/web/main.tsx", // the app's web start code: the base bundle's entry + page: (request, assets) => shell(request.nonce, assets), // the app's HTML; assets: base script, plugins file, styles +}, ctx); +// its facet (routes to Http, the WebScripts object) goes into the server's core facets +``` + +- **Base bundle** per client kind: built on the server by esbuild from `entry`, with the kind's host packages inlined + once (the app's API modules for the kind, Chord, React), content-hashed (`/_chord/web/base..js`), cached + forever (`Cache-Control: immutable`, overriding the baseline's `no-store` for these routes only); rebuilt when base + code changes (the server restarts anyway: built at start). +- **Plugin scripts:** each web facet served as a script that registers itself (`chordFacet("", "", + function (require, module, exports) { … })`, P2), at `/_chord/web/plugin/..js`, immutable; the + current set as one combined file `/_chord/web/plugins..js` (P3); stylesheets and assets of the build + likewise, under `publicPath`. +- **`WebScripts`** (a `remote` object the surface serves, params `{}`, any principal that may load the app): its state + is the current set `{ plugins: [{ name, digest, url, css? }], base }`; a rollout that ships the web kind updates it + (the plugins component's `scripts("web")` changes; the surface follows them) without waiting for pages (P5). +- Routes and the page are `Http` contributions of the surface's facet; the page route is `page: true` (navigation from + another site allowed) and public or not as the app says. + +## 2. `chord/next/web` (browser side) + +```ts +// src/web/main.tsx (the app's entry, in the base bundle) +startPage

({ + connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), + modules: { "@acme/app/v3": api, "@acme/app/v3/web": apiWeb, react: React, "react/jsx-runtime": jsx }, + core: [{ name: "@acme/web", facet: webCore }], + canRetry: async () => true, + onConnection: (state) => setBanner(state), + render: (host) => createRoot(document.getElementById("root")!).render(), +}); +``` + +- A client host (no address); the connection loop (P6, p2's `keepConnected`: reconnect 250 ms to 5 s, wake on + visibility, `online` and network change, replace a silent socket, stop on an outdated protocol (reload the page) or + when `canRetry` says no); `onConnection` reports `connecting | connected | waiting`. +- Loads the combined plugins file, evaluates each registered facet with `modules` (the `require` they get answers only + host packages and `@earendil-works/chord/next/plugins`'s browser half if any), loads them with `core` in one + `host.load`, then holds `WebScripts`: on a new set it fetches the changed scripts (` + diff --git a/packages/chord/docs/next/slice-7c.md b/packages/chord/docs/next/slice-7c.md new file mode 100644 index 000000000..f70fa38f0 --- /dev/null +++ b/packages/chord/docs/next/slice-7c.md @@ -0,0 +1,165 @@ +# Slice 7c: plugins (spec) + +Status: **ready to build** (owner, 2026-10-30: decisions "Slice 7c's points" and "Per plugin at startup and reload"; +spec reviews by Fable and Opus, `/tmp/s7c-spec-review-{fable,opus}.md`, folded in). 7c builds `chord/next/plugins`: the +code that turns plugin directories into facets, loads and reloads them per plugin, and watches them. A Chord app is +self-modifying (the owner or an agent edits a plugin while it runs), so this is core machinery, not a dev tool (no +dev/prod split). It also changes the kernel's `load` and `reload` to per-plugin failure (§4). HTTP and workers are slice +10, the web page host and the TUI slice 11: 7c builds scripts for every surface and evaluates them on Node; its build +results are plain data those slices ship. + +Read in full before writing code: this spec; `decisions.md` (2026-10-29/30: "Slice 7b's open points, as recommended", +"A plugin's npm dependencies are bundled", "Per plugin at startup and reload", "Slice 7c's points"); `design.md` (§3 +`load`/`reload`, §16, §18, §21 in full); the 7b brief `slice-7b/book.md` (chapters "A Plugin, File by File", "Building +and Loading", "Assets and Files", "Performance", "A Mini p2 from Chord"); the two spec reviews; p2's +`apps/p2/src/hosts/{bundles,plugins}.ts` and `apps/p2/docs/plugin-loading.md` §4.1, §7.1, §8, §9, §14; coding-agent's +`packages/coding-agent/src/core/package-manager.ts` (dependency installs). Code lives in +`packages/chord/src/next-node/plugins/` (Node-only: `fs`, esbuild, `vm`; the kernel in `src/next` stays portable), +exported as `./next/plugins` when v2 gets its entry point; until then tests import the source. + +## 1. A plugin on disk + +```text +plugins/notes/ + package.json "name": the plugin's name (required, npm style); "dependencies": bundled; "peerDependencies": + host packages; "chord": { "external": ["better-sqlite3"] } (optional, Node facets only) + server.ts worker.ts web.tsx tui.ts one file per facet kind the app names; default export: a facet + contract.ts store.ts NoteList.tsx notes.css assets/… plain files + node_modules/ its own npm dependencies (`installDependencies`) +``` + +- The plugin's name is its `package.json` `name`, never the directory name. A directory without a `package.json`, or + with an invalid name, is reported, not a plugin. +- Facet kinds are the app's (`["server", "worker", "web", "tui"]` in p3), each `.ts`, `.tsx`, `.js` or `.mjs`; two + files of one kind are reported (not chosen by extension). A plugin with none of the kinds (contracts only) is a + plugin with no facets: imported by others, never loaded. +- Discovery lists each root one level deep, reads only `package.json`, follows symlinks to their real paths, and keeps + the first root's plugin of a duplicate name (the other reported). + +## 2. Building + +```ts +const plugins = await discoverPlugins(["./plugins", join(dataDir, "plugins")], { kinds: KINDS }, ctx); +const built = await buildFacet(plugins[0], "worker", { hostPackages: HOST_NAMES, platform: "node" }, ctx); +// { kind: "script", plugin, facet: "worker", source, digest, inputs: [{ file, hash }], copied: [file], +// css?: { name, text }, assets: [{ name, bytes }] } | { kind: "failed", plugin, facet, errors } +``` + +One esbuild build per (plugin, kind), from cold (no kept contexts: about 10 ms per facet; p2 measured 237 ms for 25 +worker facets). A failed build is a result, never a throw. The result is plain data (strings, bytes, arrays): the same +value a worker (slice 10) or a page (slice 11) evaluates where it runs. + +| An import of | Becomes | +| --- | --- | +| the plugin's own files | inlined | +| a file of another discovered plugin (`../sessions/contract.ts`) | inlined (a copy), listed in `copied`; it may import only host packages and files of its own plugin, never packages (a build error otherwise): contracts and side-effect-free modules only (decision 4) | +| a package in the plugin's `dependencies` | inlined from its `node_modules` | +| a package in `chord.external` (Node builds) | external, `require`d at evaluation from the plugin's `node_modules`; a browser build fails | +| a host package (`options.hostPackages`, e.g. `@earendil-works/chord/next`, `react`, `react/jsx-runtime`, `@app/core`) | external, from the map at evaluation | +| `@earendil-works/chord/next/plugins` | external always (its `pluginFile` is per facet, §3) | +| a Node builtin (`node:fs`, `fs`) | external (Node builds); a browser build fails | +| CSS (browser builds) | one stylesheet per facet (`css`) | +| an asset (`import url from "./icon.svg?url"`, `.wasm`, browser builds) | a content-hashed file in `assets`; the import is its URL under the app's `options.publicPath` | +| `import.meta.url`, `__dirname`, `__filename`, a literal dynamic `import("x")`, top-level `await` | a build error pointing to `pluginFile` (§3); a computed `import(x)` cannot be seen at build and fails when it runs, through the facet's `require` naming the facet | +| anything else (a relative path into base code, a package not in `dependencies`) | a build error naming the file and the rule | + +- Output: `format: "cjs"`, wrapped `(function (require, module, exports) { … })`; TypeScript stripped, JSX automatic + (`react/jsx-runtime`); no type check; minified; an inline source map whose `sources` are relative to the plugin's + directory (so the script and its `digest`, a hash of the source, are the same on every machine and checkout). +- `inputs` are captured where esbuild loads each file (an `onLoad` that hashes the exact bytes it hands esbuild, as p2's + `captureBundleInputs`; the metafile has no hashes), absolute paths, for the watcher. + +## 3. Evaluating (Node) + +```ts +const facet = evaluateFacet(built, { hostPackages: HOST, plugin }); // returns the facet, or throws its error +``` + +- `vm.Script` under a stable filename per (plugin, kind, digest), path-like (`/chord-plugins/@pi/notes/worker.ts` + plus the digest), run in this context; the default export, checked to be a facet (a declaration error otherwise). +- **The `require` it passes answers, in order:** the host-package map (each value is what Node's `require` would give: + a CommonJS package's exports, an ESM package's namespace wrapped once with `__esModule: true`, so default and named + imports both work); `@earendil-works/chord/next/plugins` → `{ pluginFile }` bound to this plugin's directory; + `chord.external` packages and Node builtins through `createRequire(join(plugin.dir, "package.json"))`; anything else + throws naming the facet. +- **`pluginFile(path)`** gives the absolute path of a file inside the plugin's directory; `..` out of it is refused. +- **No module map, no cache:** an old facet's closures are collected once nothing holds them. The component turns V8's + compilation cache off when it is created (`v8.setFlagsFromString("--no-compilation-cache")`, once per process; an + app may pass the flag itself). +- **Stack traces** (decision Q1 (b)): `mapStack(error)` returns the error's stack with plugin scripts' positions mapped + through their source maps (a `module.SourceMap` per evaluated script, kept while the script is) to `notes/worker.ts:12`; + the app's reporters call it. Nothing global (no `Error.prepareStackTrace`, no process-wide source-map switch). + +## 4. Loading and reloading, per plugin (kernel change) + +Decision "Per plugin at startup and reload" changes `host.load` and `host.reload` (slice 7b) and the component's +`load`/`reload` helpers: + +- **`host.load(facets, ctx)`** resolves with `{ failed: [{ plugin, error }] }`: a plugin whose setup throws (or that is + refused at declaration: invalid name, a type or registry served twice, a remote type on a host without principal or + address, `checkAccepts`) is left out and reported; the others publish; its consumers wait `failed { waiting }`. `load` + never closes the host for a plugin (base code decides whether its core failing is fatal). A plugin left out is + loaded by the next reload after an edit rebuilt it (it is dirty). +- **`host.reload(facets, ctx)`**: staging per plugin. A plugin whose new code fails in staging (setup, a factory against + the recording `self`, a declaration error, `checkAccepts`) keeps its loaded facet (as if passed unchanged), and + staging runs again without its new code. **The closure rule:** a changed plugin whose recorded requests (staging + records them, for its existing objects, with their contract versions) would not bind (the same check as a real + bind, the provider's host address included) because the plugin they need was kept old, or because nobody serves the + name while any plugin of this reload failed (a failed new setup may be what would have served it: owner, 2026-10-30, + "keep"), is kept old too (repeat until stable). The report gains `failed: [{ plugin, error }]` (the same list as `load`'s; an added plugin that fails is left out); the commit applies the rest. +- **The component** builds and evaluates per plugin: a plugin whose build fails or whose evaluation throws is passed to + the host as its loaded facet at a reload, with `reload(facets, context, { failed })` naming it (the host applies the + closure rule to it and lists it in `failed`), left out at startup and reported with the host's report. + `plugins.load(host, ctx)` and `plugins.reload(host, names, ctx)` keep, per host, the facet loaded for every plugin, so + `reload` always passes the complete set (unchanged plugins as the same facet object, removed ones left out) and never + restarts an unchanged plugin. + +## 5. Watching and rollout + +```ts +const plugins = createPlugins({ roots: ["./plugins"], kinds: KINDS, hostPackages: HOST, base: ["./src"] }, ctx); +plugins.state // a JSON watchable: { dirty: ["@pi/notes"], install: ["@pi/notes"], set: false, base: false } +const next = await plugins.build(["@pi/notes"], ctx); // built results per (plugin, kind), plain data +await rollout([serverStep, ...workerSteps, ...clientSteps], next, ctx); // the app's order (decision 3) +``` + +- **The watcher** (`fs.watch` recursive over roots and `base`) ignores `node_modules` and `.git`. A changed file marks + dirty every plugin whose facets used it (`inputs`; a copied contract dirties every user) or whose directory contains + it; a changed `package.json` marks `install`; a plugin directory added or removed sets `set` (discovery runs again + at the next build); a file under `base` sets `base` (the app restarts its hosts; Chord restarts nothing). A change + during a build keeps its mark. If the watcher cannot watch (the system's limit), it reports once and the app builds + on demand. It marks, never applies. +- **`installDependencies(plugin, ctx)`**: `npm install --omit=dev --legacy-peer-deps --ignore-scripts --no-audit + --no-fund` in the plugin's real directory, the command overridable (decision 3); it clears `install`. +- **`rollout(steps, built, ctx)`**: each step is `{ name, kinds, reload(built, ctx) }` and returns the host's report; + steps run in order; a step's `reload` evaluates the facets of its kinds where its host runs (an in-process step through + the component's per-host state, §4; a remote one, slice 10/11, gets the built data). It stops at a step that rejects + (a host closed, an abort); kept plugins do not stop it. Marks clear for plugins applied on every step; a plugin kept + on any host stays dirty. + +## 6. Not in 7c + +Installing plugins from npm or git (p2 §8), project plugins with trust and snapshots (design §26 #14), a base bundle per +surface, evaluating in a browser (slice 11), shipping built scripts to workers and pages (slices 10, 11), static checks +of cross-plugin imports beyond §2's rule. + +## 7. Tests, examples, cost + +- One test file per source file; fixture plugins under `test/next-node/fixtures/plugins/`. Discovery (names, duplicates, + missing or invalid `package.json`, two files of a kind, contracts-only plugins, symlinks); build (every row of §2's + table, failures as results, relative source maps and stable digests across two checkout paths, inputs hashed at + load); evaluation (host packages as the host's instances: `instanceof ChordError` across a plugin; default and named + imports of ESM and CommonJS host packages; `pluginFile` and its `..` refusal; an import-time throw; a non-facet + export); `mapStack` naming plugin files and lines; per plugin at startup and reload (§4: a failing setup, factory, + declaration, build, evaluation; the closure rule with a provider and consumer changed together; the report's `failed` + and `failed`; `load` never closing the host); the watcher (dirty marks from inputs and directories, `install`, `set`, + `base`, a change during a build, `node_modules` ignored); `rollout` (order, stopping at a rejecting step, marks). +- **GC:** 200 reloads of an evaluated plugin facet: old closures collected, heap flat after warm-up. +- **End to end:** a server and a worker (memory links) loading plugin directories, a plugin edited on disk, built and + rolled out (server, then worker), a holder seeing one frame; a broken edit kept old on both hosts and reported. +- Examples (owner, 2026-10-30: full coverage, each a folder `NN-topic/` with the hosts in `main.ts` and real plugin + directories under `plugins/`): `35-plugins` (load, edit, build, rollout), `36-plugin-failures` (per plugin at startup + and reload, the closure rule, an abort after the commit), `37-plugin-imports` (copied contracts, dependencies, + `chord.external`, host packages, `pluginFile`, the refusals), `38-watch-and-rollout` (the marks, plugins added and + removed, a stopped rollout), `39-browser-build` (CSS, assets, the built result as plain data). Benchmark (`benchmarks.md`): cold build of a 20-plugin set, a rebuild of one facet, + evaluation per facet, heap after 100 reloads. diff --git a/packages/chord/docs/next/slice-8.md b/packages/chord/docs/next/slice-8.md index 2bd988d7a..7a2eff98b 100644 --- a/packages/chord/docs/next/slice-8.md +++ b/packages/chord/docs/next/slice-8.md @@ -1,6 +1,7 @@ # Slice 8: registries and `host.root` (spec) -Status: **ready to build** (2026-10-27; D1-D3, the registry failures and S1-S3 decided by the owner; two spec reviews, +Status: **built** (2026-10-28; build review and two fix rounds by the same four reviewers, +`/tmp/s8-build-review-*.md`, `/tmp/s8-build-rereview-*.md`). Before: ready to build (2026-10-27; D1-D3, the registry failures and S1-S3 decided by the owner; two spec reviews, `/tmp/s8-review{,2}-{astra,fable,opus,sol}.md`, found no remaining blocker; S3 refined after the second: with no value after `load`, a reader fails at once, as `self.get` of a type not served here) @@ -58,8 +59,9 @@ await host.root(async (ctx) => (await ctx.get(Notes, { session })).add(note, ctx when the table is built, an older one without an adapter or a newer one is left out and reported (`HostError` `contribution_refused { registry, plugin, version }`). - **Contributions:** a plain value, or a Chord watchable (a `writable`, a derive, a document member; another - registry's value too, a cycle of registries reading each other refused when the second one serves or contributes), followed as a dependent of the table (its change recomposes); a foreign watchable (a Durable watch) - is refused at `contribute` (declaration policy). The table's entry is `{ plugin, type?, version, value }` (`type` for + registry's value too, a cycle of registries reading each other refused when the second one serves or contributes (superseded by audit A2: caught at the first composition by the runtime guard, reported `contribution_failed`)), followed as a dependent of the table (its change recomposes), recognised by Chord's own mark; anything + else is a plain value, whatever methods it has (a harness, a conversation handle or a runtime is contributed as a plain + value, not followed; build review 2026-10-27, replacing "a foreign watchable is refused"). The table's entry is `{ plugin, type?, version, value }` (`type` for an object contributor; served types carry their plugin's name). - **`compose(table, { report })`:** pure, lazy as a derive: the registry's value is marked when its table changes (a contributor goes live or ends, a watchable contribution changes) and composed when read or, while watched, in the diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 87d395336..33a7c8ffe 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -27,15 +27,32 @@ slice spec's section citations are for orientation, not a reading limit (owner, - **L1** (`src/next/contracts`, `src/next/data`): contracts, params and identity, argument checks, principal keys, errors. `index.ts` for apps, `internal.ts` for Chord's own code. -- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28): `src/next/documents`, `state`, `host`, `jobs`, `link`, - `remote`, `store`; examples 01-25; benchmarks in `benchmarks.md`. +- **Slices 1, 2, 3a, 3b, 4a, 4b, 5, 6a, 6b, 8, 7a** (committed; 5 `4a5b45c45`, 6a and 6b 2026-10-27/28, 8 2026-10-28, 7a + 2026-10-29): `src/next/documents`, `state`, `host`, `jobs`, `link`, `remote`, `store`, `registries`; examples 01-32 + (Durable's 35-37); benchmarks in `benchmarks.md`. +- **Slice 7b** (built 2026-10-29, `slice-7b.md`): `host.reload`, staging (`host/staging.ts`), fates, adapted members + (`host/adapted.ts`), references, jobs and registries across a reload; examples 33-34; `test/next/bench/reload.ts`. +- **Slice 7c** (committed 2026-10-30, `slice-7c.md`): `src/next-node/plugins/` (discovery, builds, evaluation, + `pluginFile`, `mapStack`, `installDependencies`, the watcher, `rollout`), per-plugin `load` and `reload`; examples 35-39. +- **Slice 10** (built 2026-10-31, `slice-10.md`): `src/next-node/http/` (a `fetch` core with the security baseline, + Chord's Node listener, `nodeRequest`, plugin sockets, links), `src/next-node/workers/`, `createPluginRuntime`, kinds, + the kernel's `RouteHandle` and anonymous `host.root`, origins only from relays; examples 40-48. +- **Slice 12** (committed 2026-11-01, `slice-12.md`): the plugin model in `src/next-node/plugins/` (the app's versioned + API, per-kind files, single-file plugins, installs from git and npm, `testPlugin`); examples 49-52. +- **Slice 13** (committed 2026-11-02, `slice-13.md`): the app kit, `src/next-node/app/` (`startServer`, `startWorker`, + sessions, owner link, pairing, `Files`, `exportData`/`importData`); examples 53-55. +- **Slice 11** (committed 2026-11-03, `slice-11.md`): `src/next/surfaces/` (scripts objects, registered scripts, the + connection loop, clients, themes), `src/next-web/`, `src/next-react/`, `src/next-tui/`, `src/next-node/surfaces/`, + `src/next-node/tui/` (the launcher and its child-process sandbox); examples 56-59. - **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), context, JSON, output, SQLite drivers. ## The slices -Order (owner, 2026-10-22: routines and jobs before the store and reload). +Order (owner, 2026-10-22: routines and jobs before the store and reload; 2026-10-27: slice 8 before slice 7, whose +reload has to reload real registries; 2026-10-29: slice 7 split into 7a, 7b, 7c; the app kit as slices 10 and 11; +telemetry last, since it instruments the app kit too; 2026-11-01: the plugin model (12) and the app kit (13) before web and TUI (11), which builds into the kit). | # | Slice | In short | | --- | --- | --- | @@ -43,13 +60,19 @@ Order (owner, 2026-10-22: routines and jobs before the store and reload). | 2 | **Failures and natives** | `self.open`, activation failures and recovery, backoff, the drain, timeouts | | 3a | **Remote objects over one link** | remote contracts, principals, variants, a memory link, the message format, holds, calls, one message per commit (no relay, no drops) | | 3b | **Surviving drops** | re-holds, parking, `ref.state`, `host.hold`, `ref.retry()`, `linger` | -| 4a | **Links and relays** | host kinds, placement, routes, two-sided links, relays and principals, IPC and WebSocket, flow control, heartbeat | +| 4a | **Links and relays** | host kinds, host addresses (`placement` then), routes, two-sided links, relays and principals, IPC and WebSocket, flow control, heartbeat | | 4b | **Authority and housekeeping** | revocation, cycles across hosts, `host.references`, `host.activity`, size limits, compression | | 5 | **Routines and jobs** | `self.routines`, `self.job` | | 6a | **The store** | the store (backends, tables, queries, writes, commit queue, outbox, documents engine, migrations, export), persisted documents, `self.table`, keyed watches and retirement | | 6b | **Durable and jobs on the store** | persisted jobs, Durable's `StoreStorage`, the Chord source over Durable's views (`self.follow`) | -| 7 | **Reload and rollout** | plugin loading, staging, swaps and restarts, revisions, `accepts` in use | -| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, a `RegistryReader` for Durable, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | +| 8 | **Registries and `host.root`** | registries, `self.writable`, `retryOn`, Durable's `registryReader`, `host.root` (`chord(fn, context)` and the harness object are p3's base code) | +| 7a | **The dependency model** | requests that outlive providers, parked attempts without timers, wakes, the publication gate, getters, registries as dependencies, `parked_long` (`slice-7a.md`, `dependencies/model.md`) | +| 7b | **Reload in the kernel** | `host.reload`: staging, commit, rollback; swaps, restarts, removal and re-serving; holds and `accepts` across a reload; jobs and registries across a reload; per-name wakes (`slice-7b/` brief) | +| 7c | **Plugins** | `chord/next/plugins` (discovery, `buildFacet`, `evaluateFacet`, `pluginFile`, `installDependencies`, the watcher, `rollout`, `mapStack`; no dev/prod split; the kernel's `load`/`reload` per plugin; `slice-7c.md`) | +| 10 | **HTTP and workers** | `chord/next/http` (server, `Http` registry, responses as descriptions, WebSocket entry) and `chord/next/workers` (a worker per address, IPC, idle stop) | +| 12 | **The plugin model** | imports (relative within a repository, package names pinned across), versioned app API with Chord re-exported, per-kind API modules and contract files with the build's import check, single-file plugins, git installs, `testPlugin`, a required root (`slice-12.md`) | +| 13 | **The app kit** | `chord/next/app`: `startServer`, `startWorker`, layout, worker kinds, owner link, deny by default, sessions, CLI tokens, storage, the files plugin, export per address, plugin roots and "apply changes", base code as API, bootstrap and a start-data facet (`slice-13.md`) | +| 11 | **Web and TUI** | `chord/next/web` (page host, React hooks) and `chord/next/tui` (launcher, sandbox), from p2's code | | 9 | **Telemetry** | spans, events, metrics across hosts | The playground (`apps/chord-playground`) moves to the new code once slice 4 exists. Round-3 cuts (C3-C16) and @@ -74,15 +97,17 @@ pattern, no code. Decided cuts are noted where they apply. | 3 | park deadline (= `readyTimeout`, C13) | 3b | | 3 | `host.activity` (keep-alive counts) | 4b | | 3 | `host.root(fn, …)` for code outside objects | 8 | +| 3 | requests that outlive providers: the destination by address, the publication gate, the unbound index, the check against the selected provider only; the start barrier's first outcomes (decision (a)) | 7a | | 4 | principal type `P`, links stamp principals, principal keys in identity | 3a | | 4 | revocation (`revokedBy`, `host.revoke`, propagation to children) | 4b | | 5 | `local` types: params, members, checked arguments (Q15), results unchecked | 1 | | 5 | version and fingerprint match at acquisition; reserved member names (`then`, `release`, `state`, `retry`) | 1 | | 5 | `remote` types, variants chosen by the factory (`{ variant, members }`), identity with principal key, calls checked against the binding's variant | 3a | -| 5 | `accepts` mappings at runtime (old consumers), `contract_mismatch`/`updating` | 7 | +| 5 | `accepts` mappings at runtime (old consumers), keyed members per key, `contract_mismatch` | 7b | | 6 | lifecycle, one object per identity, lifetime from kind and params (Q-A), factory, activation, ending | 1 | | 6 | failure causes, backoff per identity, dependency recovery (R18) | 2 | | 6 | `retryOn` | 8 (first independently living source: registries) | +| 6 | parked attempts (no dependency deadline or backoff, wakes on every change, retry passing on), `FailureCause` `waiting`, `parked_long` and the authoring rules against cycles | 7a | | 6 | cycle rejection on one host (Q5) | 1 | | 6 | cycle rejection across hosts (activation chain in hold headers) | 4b | | 6 | shared store with per-principal views; worker-wide native owners; natives following a relocation | doc (1, 3, 2) | @@ -90,6 +115,7 @@ pattern, no code. Decided cuts are noted where they apply. | 7 | per-reference watches (released with their reference) | 1 | | 7 | `ctx.get` (call-scoped references) | 3a | | 7 | `ref.state` (tagged), `ref.retry()`, `host.hold`, calls parking, failed references | 3b | +| 7 | getters wait through `waiting`; `host.hold` throws only for invalid params | 7a | | 7 | `ref.variant` as discriminant, `RefIn` (C15: no `inVariant`/`expectVariant`) | 3a | | 7 | `state: false` (C8: cut) | - | | 7 | optional dependencies as registries | doc (8) | @@ -103,7 +129,7 @@ pattern, no code. Decided cuts are noted where they apply. | 9 | `initial()` checked once per definition (R8, D3) | 1 | | 9 | `persist`, `checkpointWhen` (required for persisted documents), persisted only in local empty-params objects (P3), eviction | 6a | | 9 | keyed watches (`handle.watch(key, ctx)`), `handle.at(key)`, retirement as a flag (decision "One retirement mechanism") | 6a | -| 9 | versions, `migrate`, schema fingerprints on records | 6a (in-memory migrate on reload: 7) | +| 9 | versions, `migrate`, schema fingerprints on records | 6a (in-memory migrate on reload: 7b) | | 9 | ~~facet documents, a served object's transaction spanning its facet's documents~~ (removed: C16 accepted, owner, 2026-10-26) | 1, removed in 6a | | 9 | rules across owners, reconciling routines | doc (5) | | 10 | the store: backends, tables, query runner, writes, commit queue, outbox, migrations, export/import | 6a | @@ -112,20 +138,24 @@ pattern, no code. Decided cuts are noted where they apply. | 11 | Durable's views as sources (`self.follow` over Durable's `observeXXX`) | 6b | | 12 | `self.derive`, failures per member, lazy when unwatched, `mapShared` | 1 | | 12 | container-width benchmark (R24) | 9 | -| 13 | `self.open`, close reasons, order, restart patterns (R16) | 2 (restarts on reload: 7) | -| 14 | `self.on` (W2b), `self.onLive`, `sleep`, `untilAborted` (C4 decided); the handover per object on reload | 5 (handover: 7) | -| 15 | `defineJob`, `self.job`, intent/run/finish, state documents, dedupe (Q9), `busy`, `running`, blocked `finish` (C3 decided); persistence, resume, restart limit; `accepts` in use | 5 (persistence: 6b; `accepts` and reload: 7) | +| 13 | `self.open`, close reasons, order, restart patterns (R16) | 2 (restarts on reload: 7b) | +| 14 | `self.on` (W2b), `self.onLive`, `sleep`, `untilAborted` (C4 decided); the handover per object on reload | 5 (handover: 7b) | +| 15 | `defineJob`, `self.job`, intent/run/finish, state documents, dedupe (Q9), `busy`, `running`, blocked `finish` (C3 decided); persistence, resume, restart limit; `accepts` in use | 5 (persistence: 6b; `accepts` and reload: 7b) | | 16 | `defineFacet

`, facet as declarations only (documents and routines removed in 6a) | 1, 6a | | 17 | registries: `defineRegistry`, contributions, `compose`, versions (Q8), use-site guards | 8 | +| 17 | registry readers as parked dependencies (`waiting`), `host.registry` waiting, the last good value dropped when a contribution leaves | 7a | | 18 | `constant(value)`, start data as a core object (`SessionInfo`) | 1 (core facet pattern: 4a) | | 18 | platform bindings as core objects | doc (8) | | 19 | `host.references` (presence) | 4b | | 20 | flows kept by their owner, the `steps` helper, the Questions service | doc (5); deferred decisions #9, #11 | -| 21 | loading plugins (loader D), staging, atomic commit, pure swaps, native restarts, rollout, dev loop | 7 | +| 21 | `host.reload`: staging, atomic commit, pure swaps, native restarts, removal and re-serving | 7b | +| 21 | loading plugins (loader D), dev loop; the rollout across hosts is the app's (decision "Slice 7b's open points" (3)) | 7c | | 22 | message format, header schema, unknown ids, one message per commit per link, consumers apply ops, plugin errors | 3a | -| 22 | relays, size limits per part, compression at the origin, adapted bindings get diffs | 4a (relays), 4b (limits, compression), 7 | +| 22 | relays, size limits per part, compression at the origin, adapted bindings get diffs | 4a (relays), 4b (limits, compression), 7b | | 22 | trace carriers (R5) | 9 | -| 23 | HTTP, channels, chunked transfers | after slice 9 (with the first real proof) | +| 23 | HTTP (routes, wraps, responses as descriptions, request bodies, plugin sockets), workers; channels and chunked transfers stay owner patterns until a second user | 10 | +| 23 | the app kit: `startServer`, `startWorker`, start-data objects, deny by default, sessions, owner link and token file, device pairing, files (with chunked transfers), export and import per address, plugin roots and "apply changes", optional users, invite and ephemeral-login plugins | 13 | +| 23 | web and TUI: base bundles, registered plugin scripts and the combined file, `WebScripts`/`TuiScripts`, `startPage` with the connection loop and in-place reloads, React bindings, the sandboxed terminal (launcher, sandbox, `TuiScope`), themes (`Themes`, `ThemeChoice`, CSS variables, terminal colours), CSS modules | 11 | | 24 | Durable integration: `chord(fn, context)` (Q6), a `RegistryReader` over composed registries; the harness object and its model and env registries are p3's (base code, owner, 2026-10-27) | 8 (Chord parts), p3 | | 24 | `StoreStorage` | 6b | | 27 | versions table: contracts (L1 done), registries 8, documents 6, jobs 5, link protocol 3a | as listed | @@ -143,6 +173,7 @@ Rules (owner, 2026-10-22): whoever defers something (this session, a subagent, a change; a slice's spec includes every entry of its section, and its build and review check them off; a done entry is deleted (the slice's spec and code are the record). Moved here from `carry-over.md` (owner, 2026-10-24). + ### Any slice (protocol and checks) - **Bump `PROTOCOL_VERSION`** for any message or reason change once p3 has shipped (slice 3b added `retry`, `loading` @@ -151,74 +182,26 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. slice 3b.) - **The park deadline's per-type override** is not built: a parked call waits the reference's `readyTimeout`. Build it only if a real case needs it. (Slice 3b spec.) -- Failed-check paths are TypeBox's instance paths (ambiguous for keys containing `/` or `~`); documented, not fixed. - (L1.) - No CBOR; revisit only if benchmarks show encoding as a bottleneck (slice 3a's and 4a's do not). (L2 decisions.) - **One N on every host:** a relay forwards calls at once, so a worker announcing a smaller N than the relay's could see more running calls of one origin than it accepts (a protocol error). Every host announces 256; per-link N, if ever needed, must bound the relay's forwarding too. (Slice 4a rework.) - **The Durable conversation view's size on re-hold** is a known cost until pi #11327 (windowed view) lands. (Slice 4.) -- **`map(bytes())` is refused at declaration** ("unsupported keyword `~refine`"): TypeBox's `Record` copies the bytes - schema's check as an enumerable key, which the snapshot walk refuses. An L1 bug found in slice 3a, not fixed yet. -### Slice 7: reload and rollout +### Later: reload (when a case needs it) -- **A native restart drains first** (owner, 2026-10-26): park new calls, await running calls (bounded by - `closeTimeout`), then close the natives and restart; a call outliving the drain is cut off by the seal fence. Running - job runs on a native restart: decide with the runs now owned by the object. -- **From slice 6a's second spec review** (decision "Second spec review: the majority"): remote holds and remote objects - leaving `live` (`Hold.moved` for bound and binding holds, `RemoteObject.failed` for a live object; not reachable in - 6a, P3); old table handles' version fence (old code issuing SQL for a dropped column); the routine handover fenced - per routine invocation; holds unsubscribed at seal and re-bound by member name; keyed members' versions mapped per - key; in-memory `migrate` wired to reload (6a builds the slot version check and the migrate); `rebuild` (key and - column-type changes) if a case needs it. -- **From slice 6b:** a persisted run of another job version or fingerprint stays `blocked { version }` with its record - as stored (resolve it on reload); blocked persisted runs are retried on reload; runs of jobs the code no longer - declares (below). -- **A run whose job the new code no longer declares** (slice 6a builder, 2026-10-27): 6a keeps it in its phase, - waiting, not invoked (unreachable without reload); slice 7 decides `blocked` with a reason (design §15's - `blocked { version }` family). -- **A job's new version on a live object** (slice 6a build review, 2026-10-27): 6a keeps the first declaration's job - definition in the object's records and refuses a later declaration with another version, so a host-lifetime object - (`local`, empty params) stays broken until the host restarts; slice 7 maps the records to the new version (or blocks - the old runs) on reload. +- **`rebuild` of tables** (key and column-type changes): `evolveTable` has no edit for them; build one only when a + real case needs it. (Slice 6a's second spec review; out of slice 7b's scope, `slice-7b.md` §12.) -- **Routines on reload** (from slice 5): the handover per object, bounded by `closeTimeout`; calls do not wait for it. -- **Jobs on reload** (from slice 5): `accepts` in use (args and result mapped forwards once per run; `blocked - { version }` without an adapter or for a newer version); when an older run's state document may migrate while its - old code still writes it, and an older run keeping its old handles valid until it settles; a native restart aborts - running runs (`aborted`, reason `reload`) and `finish` runs on the new code; a `finish` in flight across a reload; - (run records and state documents belong to the object: built in 6a); - blocked runs retried on reload; a job declaration removed while runs exist; a job fingerprint over args, result and - state (a schema change without a version bump); an `on` routine whose source failed and recovers after a provider - reload. -- **Instance ids on restart** (from slice 4b): a retry keeps an object's instance id; a reload that restarts an object - must give the new instance a new id, so an old chain never matches it. -- **`host.references` and removed types** (from slice 4b): a type's presence list lives from its first - `host.references(Type)` for the host's life; a reload that removes the type drops it. +### After slice 11 -- **Holds across reloads** (from slice 3a): a hold's indexes belong to its bound version. A reload that still accepts - that version keeps the hold: a pure object re-subscribes the hold through the version's mappings and sends - `[["r", value]]` for each member whose value changed; a restarted object sends a fresh snapshot for the same hold (a - second `bound` on a bound hold, which 4a's consumer applies as a re-binding). A served hold whose object leaves - `live` while its `bound` waits for a slot goes back to pending (cut from 4a, where nothing leaves `live`; owner, - 2026-10-24: slice 7 builds and tests it in `Hold.moved` and `ProviderSide.unbind`). A reload - that drops the version ends its holds with `gone` (`contract_mismatch`) before the new code serves anything. A - consumer whose own code moves to a new version re-holds (new hold id) and releases the old. -- **Old-version calls** (L1): `checkArgs` returns `{ kind: "method"; args; result? }` or `{ kind: "call"; args; call }`; - invoke the handler or the provider's `call`, and map the result back with `result` for an old-version caller. -- **Derives on reload:** derives have no names (owner, 2026-10-22); exposed derives are matched by their member name, - internal ones are rebuilt; add matching only if a real need appears. (W1.) -- **Retrying broken failures:** reload retries a broken object as `ref.retry()` does (`ChordObject.retry`: the - backoff skipped, an attempt now; holders are told `loading`). (Slices 2, 3b.) - -### Slice 8: registries and `host.root` - -- **Durable's views from other objects** (from slice 6b): the harness carried in a registry entry, so an object other - than the one that opened Durable can `self.follow` its views. - -- Specified in `slice-8.md` (decided 2026-10-27): `host.root` (D3), `retryOn` (D2), registries recomposing in the - owner's flush (W4); `host.activity` on the facet's host dropped (nothing needs it). +- **Messages and languages in the app kit** (decision "Messages and languages in the app kit", proposal + `docs/next/i18n/`): `defineMessages`, `t.message`/`t.render`, `MessagesOf`, the `Translations` registry with + fallbacks and conflicts, the locale preference per principal (default from the device, live switch, `lang`/`dir` on + the page), `useMessages` and the terminal scope's redraw, `formatTime`/`formatNumber`; tests and an example (the + notes plugin in English and German, a French translation plugin, a server message rendered per viewer). +- **Small examples** (owner, 2026-10-30: the 7c examples are complete but complex): once HTTP, workers, web and TUI + exist, a pass of small, easy-to-read examples, each showing one thing a plugin author or app does. ### Slice 9: telemetry diff --git a/packages/chord/docs/next/theming/book.md b/packages/chord/docs/next/theming/book.md new file mode 100644 index 000000000..1757d3649 --- /dev/null +++ b/packages/chord/docs/next/theming/book.md @@ -0,0 +1,223 @@ +--- +title: Looks and Themes: how an app shares its styles and components with plugins +cover: Looks / and Themes +kicker: A proposal · Chord v2 · 2026-11-02 +subtitle: One app, a notes plugin and a stranger's theme plugin, on a browser page and in a terminal +stats: + - 1 | registry for themes, shared by web and terminal + - 0 | copies of the app's components inside plugins + - 0 | page reloads to switch a theme +note: | + A proposal by the redesign session, asked for by the owner on 2026-11-02 ("can you write a small explainer about + what you envision"). It builds on decided rules: the app's versioned API split by host kind, plugins importing only + that API, registries for what plugins add (decisions of 2026-11-01 in `decisions.md`). Nothing here is built; slice + 11 (web and TUI) and slice 13 (the app kit) are where it would land. +colophon: Looks and Themes · proposal · 2026-11-02 +--- + +# Before we begin {.preface} + +Acme's notes app has a look: a sidebar, cards with rounded corners, an orange accent, a dark mode. Its own plugins +draw panels inside that look, and so should a stranger's plugin. Then someone wants a Solarized theme, and writes it as +a plugin. And the same app also runs in a terminal, where there are no CSS files and no React, only text and colours. + +Three questions follow. How does a plugin get the app's components and styles without carrying its own copy? How does +it style its own parts so they fit, without stepping on other plugins? And how does a plugin contribute a theme that +the app, every other plugin and both surfaces pick up? + +## Five things to know + +:::findings +1. **Shared components come from the app's API.** The web module of the app's API (`@acme/app/v3/web`) exports them; + it is in the base bundle once, and every plugin uses that instance. +2. **Looks are design tokens.** Colours, radii and fonts are CSS custom properties on the page (`--accent`, `--bg`); + the app's components and every plugin's CSS use them. +3. **A plugin's own CSS is scoped.** CSS modules (`notes.module.css`) keep its class names its own; the build already + collects and ships each facet's stylesheet. +4. **A theme is data in one registry.** `Themes` lives in the app's shared API module; an entry names colours for + roles (accent, background, text, ...). The web turns the active one into CSS variables, the terminal into colours. +5. **A plugin contributes a theme like anything else.** It contributes an entry to `Themes`; the picker shows it, + choosing it restyles the page and the terminal at once, editing the plugin restyles them on its rollout. +::: + +## Words used throughout + +The **app's API** is the versioned set of modules plugins may import (`@acme/app/v3`, shared by every host kind, and +`@acme/app/v3/web`, `/v3/tui` for one kind). A **host package** is a module the running process provides once to all +plugins. A **design token** is a named value of the look (the accent colour), written as a **CSS custom property** +(`--accent`) on the web. A **role** is what a colour is for (accent, background, error), as opposed to the colour +itself. A **theme** is one set of values for every role. A **registry** is an extension point plugins contribute to. + +# Shared Components {#components lede="Why plugins import the app's components instead of bundling their own."} + +A plugin's script is built without the app's API: imports of it are left as names and answered by the page at load +time (decision "Plugins import only the app's versioned API"). So when the app exports components from its web module, +every plugin gets the very same functions, from the base bundle the page loaded once: + +```ts +// src/api/v3/web.ts (the app) +export { Sheet, Button, List, Toolbar } from "../../web/components.tsx"; +export { Panels } from "./panels.web.ts"; + +// plugins/notes/web.tsx +import { defineFacet } from "@acme/app/v3"; +import { Panels, Sheet, Button } from "@acme/app/v3/web"; + +export default defineFacet((host, self) => { + self.contribute(Panels, { title: "Notes", Component: () => }); +}); +``` + +Two things follow. A plugin never ships a copy of the app's components, so a page with twenty plugins loads them once. +And when the app improves a component, every plugin shows the improvement after the app's next restart, without being +rebuilt. Components that change shape incompatibly get a new API version, as any API does. + +# Looks as Tokens {#tokens lede="One set of variables that the app's components and every plugin's CSS read."} + +The app's stylesheet, part of the base bundle, defines the look as CSS custom properties on the page's root: + +```css +:root { + --accent: #e8590c; + --bg: #111317; + --fg: #e8e8e8; + --muted: #8a8f98; + --radius: 10px; + --font: "Inter", system-ui, sans-serif; +} +``` + +The app's components use only these variables, never fixed colours. A plugin does the same in its own CSS: + +```css +/* plugins/notes/notes.module.css */ +.card { background: color-mix(in srgb, var(--bg) 92%, var(--fg)); border-radius: var(--radius); } +.title { color: var(--accent); font-family: var(--font); } +``` + +```tsx +import styles from "./notes.module.css"; +

Shopping

+``` + +Because everything reads the variables, changing the variables changes everything: the app's components and every +plugin's parts, at once, without reloading anything. That is what makes themes cheap. + +## Keeping plugins apart + +Two plugins both writing `.card { ... }` would overwrite each other's styles. CSS modules solve that: esbuild gives each +class in a `*.module.css` file a name unique to that file, and the import returns the mapping. The build already +collects each web facet's CSS into one stylesheet, served next to its script and swapped when the plugin reloads, so +nothing new is needed except the convention. Plain CSS files still work; their class names should then start with the +plugin's name. + +# The Terminal {#tui lede="The same idea without CSS: components and colour roles from the app."} + +A terminal has no stylesheets, but the same split works. The app's terminal module exports its components, built on +whichever terminal UI library the app chose, and a theme lookup by role: + +```ts +// plugins/notes/tui.ts +import { defineFacet } from "@acme/app/v3"; +import { Panels, Box, theme } from "@acme/app/v3/tui"; + +const render = (notes) => Box({ title: theme.fg("accent", "Notes"), lines: notes.map((n) => theme.fg("fg", n.text)) }); +``` + +A plugin never writes raw escape codes; it asks for a role (`accent`, `muted`, `error`), and the app's lookup turns +the active theme's colour into what the terminal supports: true colour, or the nearest of 256 colours. + +# Themes as a Registry {#themes lede="One extension point, data only, read by both surfaces."} + +A theme names a colour for each role. Roles, not surfaces: an accent is an accent on a page and in a terminal. So the +registry lives in the app's **shared** API module, and its entries are plain data, safe to import from any host kind: + +```ts +// src/api/v3/index.ts (the app) +export type Theme = { + id: string; + name: string; + mode: "dark" | "light"; + colors: { accent: string; bg: string; fg: string; muted: string; error: string; warning: string; success: string }; + web?: Readonly>; // extra CSS variables: radius, fonts, shadows + tui?: Readonly>; // terminal-only overrides: a 256-colour fallback for a role +}; +export const Themes = defineRegistry("acme.themes", { version: 1 }); +``` + +The app contributes its default themes from its own plugins, like any other contribution. Its theme picker lists the +registry's current entries; the choice is a user preference. + +:::flow wide animate | A theme from a plugin reaches the page and the terminal through one registry. +row "Plugins" muted +box sol "solarized plugin" "contributes a Theme" +box app "app's theme plugin" "contributes the defaults" +row "Shared API" accent +box reg "Themes registry" "data: roles to colours" accent +row "Surfaces" blue +box web "web page" "active theme → CSS variables on :root" blue +box tui "terminal" "active theme → colours by role" blue +arrow sol -> reg "contribute" +arrow app -> reg "contribute" +arrow reg -> web "watch" +arrow reg -> tui "watch" +::: + +**On the web**, the page watches `Themes` and the user's choice; when either changes it writes the active theme's +colours (and its `web` extras) as CSS variables on `:root`. Nothing reloads; the browser repaints. + +**In the terminal**, the app's theme lookup reads the same entry; the next redraw uses the new colours. + +# A Theme Plugin {#plugin lede="What a stranger writes to add a theme, and what happens when they edit it."} + +A theme plugin is the smallest kind of plugin there is: data, contributed. As single-file plugins (decision +"Single-file plugins"), one per surface it wants to be visible on, sharing the data: + +```ts +// plugins/solarized.web.ts +import { defineFacet, Themes } from "@acme/app/v3"; +import { solarized } from "./solarized-data.ts"; // or inline: single-file plugins import only the API +export default defineFacet((_host, self) => self.contribute(Themes, solarized)); +``` + +Since a single-file plugin may import only the app's API, a theme with two surfaces is either two files with the data +inlined, or a small directory plugin (`plugins/solarized/` with `theme.ts`, `web.ts`, `tui.ts`). + +When its author changes a colour, the watcher marks the plugin dirty; the next rollout reloads it on the server and in +every open page and terminal; the registry recomposes; pages that use Solarized repaint. No page reloads, nothing on +screen is lost. + +## When the app changes its roles + +The app adds a role (`link`) in a later release. Old themes do not name it. The registry's version guards this: the app +bumps `Themes` to version 2 and maps version 1 entries by filling the new role from the default theme. An old theme +keeps working, slightly less complete; nobody's page breaks. + +# What a Plugin Cannot Do {#limits lede="Where the app's look ends and a plugin's begins."} + +A plugin cannot change the app's base stylesheet or its components: those are base code, changed by the app's +release. It can contribute a theme (the app's looks, re-valued), style its own parts with scoped CSS that reads the +tokens, and place UI through registries like `Panels`. That line keeps a page coherent with twenty plugins from +twenty authors, and keeps one bad plugin from breaking everyone's layout. + +# Open Questions {.epilogue lede="What this proposal leaves to decide."} + +1. **The roles.** A fixed set the kit defines (accent, background, text, muted, error, warning, success) or each app's + own. Recommendation: the kit defines a base set and an app may add roles; themes fill the base set at least. +2. **Where `Themes` lives.** In the app kit (every kit app gets it and the CSS-variable writer) or in each app. + Recommendation: the kit ships it, with the picker's preference and the writer; apps add roles and their defaults. +3. **Light and dark.** A theme carries one mode; following the operating system's preference picks the user's chosen + dark or light theme. Recommendation: yes, two choices per user (one dark, one light). +4. **Fonts and assets in themes.** Allowed through `web` extras with URLs to the plugin's assets (the build serves + them), or colours only. Recommendation: allowed, under the page's CSP (`font-src`, `img-src` self). + +# Reference {.appendix} + +| Piece | Where | What | +| --- | --- | --- | +| shared components (web) | `@acme/app/v3/web` | React components, one instance in the base bundle | +| shared components (terminal) | `@acme/app/v3/tui` | the app's terminal components and `theme` lookup | +| design tokens | the app's base stylesheet | CSS custom properties on `:root` | +| plugin styles | `*.module.css` in the plugin | scoped class names, shipped with the facet's script | +| `Themes` | `@acme/app/v3` (shared) | registry of theme data, read by both surfaces | +| theme plugin | `plugins/.web.ts`, `.tui.ts` or a directory | contributes a `Theme` | diff --git a/packages/chord/docs/next/theming/index.html b/packages/chord/docs/next/theming/index.html new file mode 100644 index 000000000..b6db7c4d7 --- /dev/null +++ b/packages/chord/docs/next/theming/index.html @@ -0,0 +1,371 @@ + + +Looks and Themes: how an app shares its styles and components with plugins + + + + +
+
+
A proposal · Chord v2 · 2026-11-02
+

Looks
and Themes

+

One app, a notes plugin and a stranger's theme plugin, on a browser page and in a terminal

+
1registry for themes, shared by web and terminal
0copies of the app's components inside plugins
0page reloads to switch a theme
+

A proposal by the redesign session, asked for by the owner on 2026-11-02 ("can you write a small explainer about +what you envision"). It builds on decided rules: the app's versioned API split by host kind, plugins importing only +that API, registries for what plugins add (decisions of 2026-11-01 in decisions.md). Nothing here is built; slice +11 (web and TUI) and slice 13 (the app kit) are where it would land.

+
+
+ + +
+

Before we begin

+

Acme's notes app has a look: a sidebar, cards with rounded corners, an orange accent, a dark mode. Its own plugins +draw panels inside that look, and so should a stranger's plugin. Then someone wants a Solarized theme, and writes it as +a plugin. And the same app also runs in a terminal, where there are no CSS files and no React, only text and colours.

+

Three questions follow. How does a plugin get the app's components and styles without carrying its own copy? How does +it style its own parts so they fit, without stepping on other plugins? And how does a plugin contribute a theme that +the app, every other plugin and both surfaces pick up?

+

Five things to know

+
    +
  1. Shared components come from the app's API. The web module of the app's API (@acme/app/v3/web) exports them; +it is in the base bundle once, and every plugin uses that instance.
    2. +
  2. Looks are design tokens. Colours, radii and fonts are CSS custom properties on the page (--accent, --bg); +the app's components and every plugin's CSS use them.
    3. +
  3. A plugin's own CSS is scoped. CSS modules (notes.module.css) keep its class names its own; the build already +collects and ships each facet's stylesheet.
    4. +
  4. A theme is data in one registry. Themes lives in the app's shared API module; an entry names colours for +roles (accent, background, text, ...). The web turns the active one into CSS variables, the terminal into colours.
    5. +
  5. A plugin contributes a theme like anything else. It contributes an entry to Themes; the picker shows it, +choosing it restyles the page and the terminal at once, editing the plugin restyles them on its rollout.
    6. +
+

Words used throughout

+

The app's API is the versioned set of modules plugins may import (@acme/app/v3, shared by every host kind, and +@acme/app/v3/web, /v3/tui for one kind). A host package is a module the running process provides once to all +plugins. A design token is a named value of the look (the accent colour), written as a CSS custom property +(--accent) on the web. A role is what a colour is for (accent, background, error), as opposed to the colour +itself. A theme is one set of values for every role. A registry is an extension point plugins contribute to.

+
+ +
I

Shared Components

Why plugins import the app's components instead of bundling their own.

+

A plugin's script is built without the app's API: imports of it are left as names and answered by the page at load +time (decision "Plugins import only the app's versioned API"). So when the app exports components from its web module, +every plugin gets the very same functions, from the base bundle the page loaded once:

+
// src/api/v3/web.ts (the app)
+export { Sheet, Button, List, Toolbar } from "../../web/components.tsx";
+export { Panels } from "./panels.web.ts";
+
+// plugins/notes/web.tsx
+import { defineFacet } from "@acme/app/v3";
+import { Panels, Sheet, Button } from "@acme/app/v3/web";
+
+export default defineFacet((host, self) => {
+	self.contribute(Panels, { title: "Notes", Component: () => <Sheet><Button>New note</Button></Sheet> });
+});
+

Two things follow. A plugin never ships a copy of the app's components, so a page with twenty plugins loads them once. +And when the app improves a component, every plugin shows the improvement after the app's next restart, without being +rebuilt. Components that change shape incompatibly get a new API version, as any API does.

+
+ +
II

Looks as Tokens

One set of variables that the app's components and every plugin's CSS read.

+

The app's stylesheet, part of the base bundle, defines the look as CSS custom properties on the page's root:

+
:root {
+	--accent: #e8590c;
+	--bg: #111317;
+	--fg: #e8e8e8;
+	--muted: #8a8f98;
+	--radius: 10px;
+	--font: "Inter", system-ui, sans-serif;
+}
+

The app's components use only these variables, never fixed colours. A plugin does the same in its own CSS:

+
/* plugins/notes/notes.module.css */
+.card { background: color-mix(in srgb, var(--bg) 92%, var(--fg)); border-radius: var(--radius); }
+.title { color: var(--accent); font-family: var(--font); }
+
import styles from "./notes.module.css";
+<div className={styles.card}><h2 className={styles.title}>Shopping</h2></div>
+

Because everything reads the variables, changing the variables changes everything: the app's components and every +plugin's parts, at once, without reloading anything. That is what makes themes cheap.

+

Keeping plugins apart

+

Two plugins both writing .card { ... } would overwrite each other's styles. CSS modules solve that: esbuild gives each +class in a *.module.css file a name unique to that file, and the import returns the mapping. The build already +collects each web facet's CSS into one stylesheet, served next to its script and swapped when the plugin reloads, so +nothing new is needed except the convention. Plain CSS files still work; their class names should then start with the +plugin's name.

+
+ +
III

The Terminal

The same idea without CSS: components and colour roles from the app.

+

A terminal has no stylesheets, but the same split works. The app's terminal module exports its components, built on +whichever terminal UI library the app chose, and a theme lookup by role:

+
// plugins/notes/tui.ts
+import { defineFacet } from "@acme/app/v3";
+import { Panels, Box, theme } from "@acme/app/v3/tui";
+
+const render = (notes) => Box({ title: theme.fg("accent", "Notes"), lines: notes.map((n) => theme.fg("fg", n.text)) });
+

A plugin never writes raw escape codes; it asks for a role (accent, muted, error), and the app's lookup turns +the active theme's colour into what the terminal supports: true colour, or the nearest of 256 colours.

+
+ +
IV

Themes as a Registry

One extension point, data only, read by both surfaces.

+

A theme names a colour for each role. Roles, not surfaces: an accent is an accent on a page and in a terminal. So the +registry lives in the app's shared API module, and its entries are plain data, safe to import from any host kind:

+
// src/api/v3/index.ts (the app)
+export type Theme = {
+	id: string;
+	name: string;
+	mode: "dark" | "light";
+	colors: { accent: string; bg: string; fg: string; muted: string; error: string; warning: string; success: string };
+	web?: Readonly<Record<string, string>>;   // extra CSS variables: radius, fonts, shadows
+	tui?: Readonly<Record<string, string>>;   // terminal-only overrides: a 256-colour fallback for a role
+};
+export const Themes = defineRegistry<Theme, readonly Theme[]>("acme.themes", { version: 1 });
+

The app contributes its default themes from its own plugins, like any other contribution. Its theme picker lists the +registry's current entries; the choice is a user preference.

+
PLUGINSSHARED APISURFACESsolarized plugincontributes a Themeapp's theme plugincontributes the defaultsThemes registrydata: roles to coloursweb pageactive theme → CSS variables on :rootterminalactive theme → colours by rolecontributecontributewatchwatch
Figure 1. A theme from a plugin reaches the page and the terminal through one registry.
+

On the web, the page watches Themes and the user's choice; when either changes it writes the active theme's +colours (and its web extras) as CSS variables on :root. Nothing reloads; the browser repaints.

+

In the terminal, the app's theme lookup reads the same entry; the next redraw uses the new colours.

+
+ +
V

A Theme Plugin

What a stranger writes to add a theme, and what happens when they edit it.

+

A theme plugin is the smallest kind of plugin there is: data, contributed. As single-file plugins (decision +"Single-file plugins"), one per surface it wants to be visible on, sharing the data:

+
// plugins/solarized.web.ts
+import { defineFacet, Themes } from "@acme/app/v3";
+import { solarized } from "./solarized-data.ts";   // or inline: single-file plugins import only the API
+export default defineFacet((_host, self) => self.contribute(Themes, solarized));
+

Since a single-file plugin may import only the app's API, a theme with two surfaces is either two files with the data +inlined, or a small directory plugin (plugins/solarized/ with theme.ts, web.ts, tui.ts).

+

When its author changes a colour, the watcher marks the plugin dirty; the next rollout reloads it on the server and in +every open page and terminal; the registry recomposes; pages that use Solarized repaint. No page reloads, nothing on +screen is lost.

+

When the app changes its roles

+

The app adds a role (link) in a later release. Old themes do not name it. The registry's version guards this: the app +bumps Themes to version 2 and maps version 1 entries by filling the new role from the default theme. An old theme +keeps working, slightly less complete; nobody's page breaks.

+
+ +
VI

What a Plugin Cannot Do

Where the app's look ends and a plugin's begins.

+

A plugin cannot change the app's base stylesheet or its components: those are base code, changed by the app's +release. It can contribute a theme (the app's looks, re-valued), style its own parts with scoped CSS that reads the +tokens, and place UI through registries like Panels. That line keeps a page coherent with twenty plugins from +twenty authors, and keeps one bad plugin from breaking everyone's layout.

+
+ +
∴

Open Questions

What this proposal leaves to decide.

+
    +
  1. The roles. A fixed set the kit defines (accent, background, text, muted, error, warning, success) or each app's +own. Recommendation: the kit defines a base set and an app may add roles; themes fill the base set at least.
    2. +
  2. Where Themes lives. In the app kit (every kit app gets it and the CSS-variable writer) or in each app. +Recommendation: the kit ships it, with the picker's preference and the writer; apps add roles and their defaults.
    3. +
  3. Light and dark. A theme carries one mode; following the operating system's preference picks the user's chosen +dark or light theme. Recommendation: yes, two choices per user (one dark, one light).
    4. +
  4. Fonts and assets in themes. Allowed through web extras with URLs to the plugin's assets (the build serves +them), or colours only. Recommendation: allowed, under the page's CSP (font-src, img-src self).
    5. +
+
+ +
A

Reference

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PieceWhereWhat
shared components (web)@acme/app/v3/webReact components, one instance in the base bundle
shared components (terminal)@acme/app/v3/tuithe app's terminal components and theme lookup
design tokensthe app's base stylesheetCSS custom properties on :root
plugin styles*.module.css in the pluginscoped class names, shipped with the facet's script
Themes@acme/app/v3 (shared)registry of theme data, read by both surfaces
theme pluginplugins/<name>.web.ts, .tui.ts or a directorycontributes a Theme
+
+ +

Looks and Themes · proposal · 2026-11-02

+
+ + diff --git a/packages/chord/docs/spec.md b/packages/chord/docs/spec.md index ca9c93b38..4c261cb54 100644 --- a/packages/chord/docs/spec.md +++ b/packages/chord/docs/spec.md @@ -3,7 +3,7 @@ This document specifies the runtime in `src`. Where it and the code disagree, the disagreement is a bug. [Known limitations and decisions](#12-known-limitations-and-decisions) lists what the runtime does not do yet and the choices the sections below leave open. Hard constraints: -- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins plus `typebox` for schemas. No other Pi packages. +- Chord's runtime depends only on its own `delta` module (tracker, applyImmutable, encoder/decoder, diff) and Node/browser built-ins, plus `@earendil-works/pi-telemetry` for telemetry and `@earendil-works/pi-schema` for schemas. No other Pi packages. - Built-in plugins are trusted, loaded from source in-process. The plugin loader uses dynamic `import()` with a cache-busting query (owner-approved exception, `TODO(loader)`: replace with an isolating, evicting loader; old module copies leak; TUI plugins need isolation). - Hot reload is required. Host restart only for Chord/runtime code changes or host-level crash recovery. @@ -29,23 +29,23 @@ Contracts are runtime values for every type (shipped bundles have no TypeScript; ```ts export const Chat = remote("pi.chat", { version: 1, // contract version: a plain number, bumped only on breaking changes - input: Type.Object({ session: Type.String(), conversation: Type.Number() }), + input: obj({ session: str(), conversation: num() }), host: hostFamily("session:", "session"), // where its objects live: `session:` + input.session variants: { viewer: { state: { view: ViewSchema } }, participant: { state: { view: ViewSchema }, - methods: { send: method(SendSchema, Type.Object({ submissionId: Type.Number() })) }, + methods: { send: method(SendSchema, obj({ submissionId: num() })) }, }, }, }); export const Index = local("pi.index", { version: 1, - input: Type.Object({ session: Type.String() }), + input: obj({ session: str() }), members: { - search: method(Type.Object({ query: Type.String() }), Type.Array(Hit)), - onChange: method(Type.Function([Change], Type.Void()), Unsubscribe), // non-JSON allowed for locals + search: method(obj({ query: str() }), arr(Hit)), + onChange: method({ type: "function", parameters: [Change], returnType: { type: "void" } }, Unsubscribe), // non-JSON allowed for locals harness: opaque("pi.durable-harness@1"), // native object, by name+version }, }); @@ -81,7 +81,7 @@ Factories are synchronous; returning a promise throws. A declaration has a name host.serve(StatusStore, ({ session }, _host, self) => { const status = self.state("status", StatusSchema, { running: false, tasks: 0 }, // schema required, initial value { version: 2, migrate: (old, from) => upgrade(old, from) }); // per cell; default version 1 - const errors = self.state("errors", Type.Array(Type.String()), []); + const errors = self.state("errors", arr(str()), []); const catalog = self.get(Catalog, {}); // reference; identity = canonical address (+ at) const watch = self.resource("watch", { version: 1, deps: [catalog], exclusive: false, @@ -278,8 +278,8 @@ The tests cover at least: lifetime/references/linger/forced disposal/gone; activ - **Joining during a reload.** Participants are the child hosts in step when a reload starts. A child whose hello arrives while it runs is decided when it ends: in step, or reconciled to the committed code version; a child already behind is fenced at once, so the calls its roots make meanwhile are admitted in the parent's code version. A reconciliation that is running when a reload starts syncs its child only after that reload ended, against the code version it committed. A reconciliation announces the parent's retired code versions before its `prepare`. - **Two-version rule.** An object of a changed type created after a commit has only new code; a previous-code-version root calling it gets `superseded`. - **Validation.** Remote calls are validated at reference dispatch, in-host too: a direct binding against the provider's schema, a binding through an adapter against the consumer's. A provider validates each snapshot and reset of a remote state member once per (re)subscription, against the consumer's contract through an `accepts` adapter and the provider's otherwise; ops are validated only in dev mode. At reload, the new provider's description must cover the old provider's when the version is unchanged, in addition to the check against live consumers and facets' `uses`. -- **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; with TypeBox, use `Type.Optional(Type.X({ default }))` to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. -- **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. +- **Contracts.** Local contracts list their members, so a loading reference knows whether a member is a method, state or opaque. Canonical identity omits input fields equal to their default; use `opt(str({ default }))` (pi-schema) to let callers omit them. Provider variants must cover consumer variants; members the consumer's contract does not list are hidden. Schema change detection compares canonical JSON, so reordering `anyOf` branches counts as a change. Fingerprints use a non-cryptographic 64-bit hash: synchronous and browser-safe; contracts are trusted. +- **Principals.** A root ctx at ingress acts as its principal (`host.root(fn, { principal })`); contexts inside objects act as `host`. The acceptor decides a link's kind and name (`accept(link, { principal, kind, name })`); only configured host links are addressable, carry roots and may vote, and control frames are accepted only from the parent. Clients never carry roots or code versions. Relaying needs a matching `route`; `authorize(principal, hostName)` runs before `resolve` and again after, and revoking or re-stamping a principal closes its relays at once. The host `resolve` returns is admitted before that second check, so a refused, revoked or released relay never leaves a spawned host's link unread. `invalidatePrincipal(match)` re-stamps or closes matching links and propagates pattern matches to child hosts. - **Cells and readables.** A value returned from `change` replaces the value and is always validated; `replace` transfers ownership without copying; by default (`{ diff }` overrides) it publishes a diff when the new value reuses a container of the old one at the same key, else `r`. `subscribe(fn)` delivers values, starting with the current one; `events(fn)` delivers `snap`, then `ops`, or `reset`. Reference state members are stable readables that throw `not_ready` while loading and `not_in_variant` outside the variant, and reset when their source moves; a listener added before they are readable gets its snapshot then. Remote state members stay hydrated while the reference is held. - **References and calls.** A reference's identity is its address; a second `self.get` of the same address in one run returns the same reference. A reference call without ctx, or with a task ctx, starts a fresh root in the current code version; a ctx whose call or root already settled is refused (`aborted/expired`), one from another host `invalid/foreign_ctx`. Calling a reference inside a factory throws `not_ready`. - **Resources and disposal.** New and replaced exclusive resources open at the gate, and so do non-exclusive resources of the same object that depend on them. A new object for an address that is still being disposed waits for that disposal. A resource open that completes after its signal aborted closes its value at once. diff --git a/packages/chord/docs/tutorial.md b/packages/chord/docs/tutorial.md index fe16dce4c..65a6334ed 100644 --- a/packages/chord/docs/tutorial.md +++ b/packages/chord/docs/tutorial.md @@ -45,11 +45,12 @@ shuts down its objects and connections. Before callers can use an object, they need to agree on its name and members. That agreement is a **contract**: a runtime value that also gives TypeScript its -types. A **schema** describes a value's shape. We use TypeBox's `Type` to write -schemas. `method(args, result)` declares a method by its argument and result -schemas: `method()` takes no arguments and returns nothing, and -`method(undefined, Type.String())`, used below, takes no arguments and returns a -string. +types. A **schema** describes a value's shape: plain JSON Schema, which we write +with the builders of `@earendil-works/pi-schema` (`obj`, `str`, `num`, ...; +`{ open: true }` lets an object carry fields it does not name). +`method(args, result)` declares a method by its argument and result schemas: +`method()` takes no arguments and returns nothing, and +`method(undefined, str())`, used below, takes no arguments and returns a string. Start with a label for a named list. `local` means only code inside this host can reach it. The input `{ list: "family" }` identifies which list we want; @@ -59,8 +60,8 @@ it is not an argument to `label`. ```ts const TodoLabel = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { label: method(undefined, Type.String()) }, + input: obj({ list: str() }, { open: true }), + members: { label: method(undefined, str()) }, }); ``` @@ -116,17 +117,17 @@ name, because it holds nothing a reload must keep. ```ts -const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); -const Rows = Type.Array(Todo); +const Todo = obj({ text: str(), done: bool() }, { open: true }); +const Rows = arr(Todo); const TodoStore = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), members: { - label: method(undefined, Type.String()), + label: method(undefined, str()), rows: Rows, - openCount: Type.Number(), - add: method(Type.Object({ text: Type.String() })), - finish: method(Type.Object({ index: Type.Integer() })), + openCount: num(), + add: method(obj({ text: str() }, { open: true })), + finish: method(obj({ index: int() }, { open: true })), reset: method(Rows), }, }); @@ -230,13 +231,13 @@ to a declared resource, without reading the resource during the factory. ```ts const Database = local("tutorial.database", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, }); const SavedTodos = local("tutorial.saved-todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { add: method(Type.Object({ text: Type.String() })) }, + input: obj({ list: str() }, { open: true }), + members: { add: method(obj({ text: str() }, { open: true })) }, }); const host = createHost({ self: "server" }); let opens = 0; @@ -338,13 +339,13 @@ puts every `Todos` object on the host named `server`. Section 6 uses that. ```ts const Todos = remote("tutorial.todo-view", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { rows: Rows } }, editor: { state: { rows: Rows }, - methods: { add: method(Type.Object({ text: Type.String() })) }, + methods: { add: method(obj({ text: str() }, { open: true })) }, }, }, }); @@ -499,8 +500,8 @@ test; they are not stored application state. ```ts const Activity = local("tutorial.activity", { version: 1, - input: Type.Object({}), - members: { seen: Type.Number() }, + input: obj({}, { open: true }), + members: { seen: num() }, }); const host = createHost({ self: "server" }); host.serve(TodoStore, todoFactory); @@ -509,7 +510,7 @@ const seeded = Promise.withResolvers(); const stopped = Promise.withResolvers(); host.serve(Activity, (_input, _principal, self) => { const todos = self.get(TodoStore, { list: "family" }); - const seen = self.state("seen", Type.Number(), 0); + const seen = self.state("seen", num(), 0); self.on("rows", todos.rows, (rows, ctx) => { seen.replace(ctx, rows?.length ?? 0); if (rows?.length === 1) observed.resolve(); @@ -570,7 +571,7 @@ default lifetime, `"referenced"`, are still created only when referenced. ```ts const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); -const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); +const Menu = local("tutorial.menu", { version: 1, input: obj({}, { open: true }) }); let click: (() => Promise) | undefined; const storeFacet: Facet = { name: "todo-store", @@ -745,7 +746,7 @@ const { label: _label, ...members } = TodoStore.def.members; const RenamedStore = local("tutorial.todos", { version: 2, input: TodoStore.def.input, - members: { ...members, heading: method(undefined, Type.String()) }, + members: { ...members, heading: method(undefined, str()) }, }); await host.reload([ { @@ -890,7 +891,7 @@ tutorial's source-matching check, not by the application. ```ts import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { type Static, Type } from "typebox"; +import { arr, bool, int, num, obj, type Static, str } from "@earendil-works/pi-schema"; import { it, onTestFinished } from "vitest"; import { type Ctx, diff --git a/packages/chord/package.json b/packages/chord/package.json index 895c16523..c61aef777 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/chord", - "version": "1.4.0", + "version": "1.5.0", "description": "Typed host runtime for plugins, contracts, replicated state, and RPC over JSON links", "type": "module", "main": "./dist/index.js", @@ -67,7 +67,7 @@ "src/delta/README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -90,12 +90,31 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/pi-telemetry": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0" + }, + "peerDependencies": { + "esbuild": "0.28.2", + "react": "19.3.0", + "ws": "8.21.0" + }, + "peerDependenciesMeta": { + "esbuild": { + "optional": true + }, + "react": { + "optional": true + }, + "ws": { + "optional": true + } }, "devDependencies": { + "@types/react": "19.2.14", + "@types/react-dom": "19.2.3", "@types/ws": "8.18.2", - "shx": "0.3.4", + "react": "19.3.0", + "react-dom": "19.3.0", "vitest": "4.1.12", "ws": "8.21.0" } diff --git a/packages/chord/src/cell.ts b/packages/chord/src/cell.ts index 65d391128..d520258e3 100644 --- a/packages/chord/src/cell.ts +++ b/packages/chord/src/cell.ts @@ -1,4 +1,4 @@ -import type { TSchema } from "typebox"; +import type { Schema } from "@earendil-works/pi-schema"; import { type Draft, diffRevisions, type JsonValue, type Op, type Tracker, track } from "./delta/index.ts"; import { ChordError } from "./errors.ts"; import { enqueue, holdPublications } from "./publish.ts"; @@ -87,7 +87,7 @@ export abstract class Source implements Readable { readonly seq = ++sequence; readonly dependents = new Set>(); /** Schemas of remote state members currently exposing this source (checked per write in dev mode). */ - schemas: readonly TSchema[] = []; + schemas: readonly Schema[] = []; protected readonly options: SourceOptions; protected readonly fanout: Fanout; @@ -114,7 +114,7 @@ export abstract class Source implements Readable { for (const schema of this.schemas) assertValid("state", `state ${this.key}`, schema, value); } - protected ownSchemas(): readonly TSchema[] { + protected ownSchemas(): readonly Schema[] { return []; } } @@ -149,7 +149,7 @@ function unbox(ops: readonly Op[], value: unknown): Op[] { /** A state cell holding any JSON value, validated against its schema at creation and (dev mode) on writes. */ export class Cell extends Source { /** Replaced at a reload commit that changes the schema (state-version bump). */ - schema: TSchema; + schema: Schema; #tracker: Tracker>; readonly #guard: WriteGuard; /** False until `initialize` for a cell declared with `init`: reads and writes throw `not_ready`. */ @@ -161,7 +161,7 @@ export class Cell extends Source { */ constructor( key: string, - schema: TSchema, + schema: Schema, initial: { readonly value: T } | undefined, guard: WriteGuard, options: SourceOptions, @@ -182,7 +182,7 @@ export class Cell extends Source { this.#initialized = true; } - protected override ownSchemas(): readonly TSchema[] { + protected override ownSchemas(): readonly Schema[] { return [this.schema]; } @@ -238,7 +238,7 @@ export class Cell extends Source { * Installs a migrated value and its schema (reload commit). The caller validated the value and the derivations * over it against every schema that will apply, so nothing is checked here. Subscribers receive a reset. */ - reset(value: T, schema: TSchema): void { + reset(value: T, schema: Schema): void { this.schema = schema; this.#publish(this.#tracker.prepareReplace({ v: value }), "reset", false); } diff --git a/packages/chord/src/contract.ts b/packages/chord/src/contract.ts index c2ebd6b96..1a6d64031 100644 --- a/packages/chord/src/contract.ts +++ b/packages/chord/src/contract.ts @@ -1,15 +1,14 @@ -import { type Static, type TSchema, Type } from "typebox"; -import { Value } from "typebox/value"; +import { int, lit, map, obj, opt, type Schema, type Static, str, union } from "@earendil-works/pi-schema"; import { ChordError, invalid } from "./errors.ts"; -import { canonicalJson, fits, hash64 } from "./schema.ts"; +import { canonicalJson, fits, hash64, isValid } from "./schema.ts"; import type { Ctx, Exposed, Readable, RefBase, StateSource } from "./types.ts"; const MEMBER = Symbol("chord.member"); /** A method's arguments and result; `undefined`: it takes no arguments, or returns nothing (void). */ export interface MethodSpec< - A extends TSchema | undefined = TSchema | undefined, - R extends TSchema | undefined = TSchema | undefined, + A extends Schema | undefined = Schema | undefined, + R extends Schema | undefined = Schema | undefined, > { readonly [MEMBER]: "method"; readonly args: A; @@ -29,9 +28,9 @@ export interface OpaqueSpec { * for a method without arguments that returns something. */ export function method(): MethodSpec; -export function method(args: A): MethodSpec; -export function method(args: A, result: R): MethodSpec; -export function method(args?: TSchema, result?: TSchema): MethodSpec { +export function method(args: A): MethodSpec; +export function method(args: A, result: R): MethodSpec; +export function method(args?: Schema, result?: Schema): MethodSpec { return { [MEMBER]: "method", args, result }; } @@ -42,7 +41,7 @@ export const opaque = (id: string): OpaqueSpec => { }; export interface VariantDef { - readonly state?: Readonly>; + readonly state?: Readonly>; readonly methods?: Readonly>; } @@ -102,63 +101,59 @@ export const placementText = (placement: Placement): string => export interface RemoteDef { readonly version: number; - readonly input: TSchema; + readonly input: Schema; /** Where its objects live: `hostNamed(…)` or `hostFamily(…)`. Part of the contract (and its fingerprint). */ readonly host: Placement; readonly variants: Readonly>; } /** A local member: a method, an opaque native object, or a state member described by its schema. */ -export type LocalMember = MethodSpec | OpaqueSpec | TSchema; +export type LocalMember = MethodSpec | OpaqueSpec | Schema; export interface LocalDef { readonly version: number; - readonly input: TSchema; + readonly input: Schema; readonly members?: Readonly>; } // A description travels as a reference's contract token (wire.ts), so it is declared once, as the schema that validates it. -// `TSchema` is an empty interface: any object is one, which is all a peer's schema is checked for here. -const SchemaObject = Type.Unsafe(Type.Object({})); +// Any object is a schema, which is all a peer's schema is checked for here. +const SchemaObject = obj({}, { open: true }) as Schema; -const MemberDescriptionSchema = Type.Union([ +const MemberDescriptionSchema = union([ // Absent `args`: the method takes none. Absent `result`: it returns nothing. - Type.Object({ - kind: Type.Literal("method"), - args: Type.Optional(SchemaObject), - result: Type.Optional(SchemaObject), - }), - Type.Object({ kind: Type.Literal("state"), schema: SchemaObject }), - Type.Object({ kind: Type.Literal("opaque"), id: Type.String() }), + obj({ kind: lit("method"), args: opt(SchemaObject), result: opt(SchemaObject) }, { open: true }), + obj({ kind: lit("state"), schema: SchemaObject }, { open: true }), + obj({ kind: lit("opaque"), id: str() }, { open: true }), ]); export type MemberDescription = Static; /** Local contracts have one unnamed variant. */ export const LOCAL_VARIANT = ""; -const PlacementSchema = Type.Union([ - Type.Object({ name: Type.String({ minLength: 1 }) }, { additionalProperties: false }), - Type.Object( - { prefix: Type.String({ minLength: 1 }), key: Type.String({ minLength: 1 }) }, - { additionalProperties: false }, - ), +const PlacementSchema = union([ + obj({ name: str({ minLength: 1 }) }), + obj({ prefix: str({ minLength: 1 }), key: str({ minLength: 1 }) }), ]); -export const DescriptionSchema = Type.Object({ - name: Type.String({ minLength: 1 }), - scope: Type.Union([Type.Literal("remote"), Type.Literal("local")]), - version: Type.Integer(), - input: SchemaObject, - /** Remote types only (required there): where their objects live. */ - host: Type.Optional(PlacementSchema), - variants: Type.Record(Type.String(), Type.Record(Type.String(), MemberDescriptionSchema)), -}); +export const DescriptionSchema = obj( + { + name: str({ minLength: 1 }), + scope: union([lit("remote"), lit("local")]), + version: int(), + input: SchemaObject, + /** Remote types only (required there): where their objects live. */ + host: opt(PlacementSchema), + variants: map(map(MemberDescriptionSchema)), + }, + { open: true }, +); export type Description = Static; interface ContractBase { readonly name: string; readonly version: number; - readonly input: TSchema; + readonly input: Schema; readonly description: Description; readonly fingerprint: string; } @@ -204,7 +199,7 @@ function describeMember(name: string, member: unknown, scope: "remote" | "local" if (scope === "remote") throw new TypeError(`remote contracts cannot expose opaque members: ${name}`); return { kind: "opaque", id: (member as OpaqueSpec).id }; } - return { kind: "state", schema: member as TSchema }; + return { kind: "state", schema: member as Schema }; } function finish(contract: Omit): C { @@ -222,7 +217,7 @@ export function remote( ): RemoteContract { checkHeader(name, def.version); const placement = def.host as Placement | undefined; - if (placement === undefined || !Value.Check(PlacementSchema, placement)) + if (placement === undefined || !isValid(PlacementSchema, placement)) throw new TypeError(`remote contract ${name} needs \`host\`: hostNamed(…) or hostFamily(…)`); // A JSON copy: the description travels, and a placement object may carry nothing else. const host: Placement = @@ -353,23 +348,23 @@ export function bind(provider: AnyContract, accepts: Accepts | undefined, consum // ─── Static types ──────────────────────────────────────────────────────────── type Empty = Record; -type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; -type StaticOf = S extends TSchema ? Static : never; +type StateOf = V extends { readonly state: infer S extends Readonly> } ? S : Empty; +type StaticOf = S extends Schema ? Static : never; type MethodsOf = V extends { readonly methods: infer M extends Readonly> } ? M : Empty; /** What a handler returns: the result's value, or nothing for a void method. */ -type Returned = [R] extends [TSchema] ? Static | Promise> : void | Promise; +type Returned = [R] extends [Schema] ? Static | Promise> : void | Promise; /** What a call resolves to. */ -type Resolved = [R] extends [TSchema] ? Static : undefined; +type Resolved = [R] extends [Schema] ? Static : undefined; /** A method's handler; `P` is the app's principal type (`ctx.principal`). */ export type MethodImplementation = M extends MethodSpec - ? [A] extends [TSchema] + ? [A] extends [Schema] ? (args: Static, ctx: Ctx

) => Returned : (ctx: Ctx

) => Returned : never; export type Call = M extends MethodSpec - ? [A] extends [TSchema] + ? [A] extends [Schema] ? (args: Static, ctx?: Ctx) => Promise> : (ctx?: Ctx) => Promise> : never; @@ -402,21 +397,21 @@ type LocalImplementationMember = M extends MethodSpec ? MethodImplementation : M extends OpaqueSpec ? T | Exposed - : M extends TSchema + : M extends Schema ? StateSource> : never; type LocalRefMember = M extends MethodSpec ? Call : M extends OpaqueSpec ? T | undefined - : M extends TSchema + : M extends Schema ? Readable> : never; type LocalMembers = D extends { readonly members: infer M extends Readonly> } ? M : Empty; -export type InputOf = C extends { readonly def: { readonly input: infer I extends TSchema } } ? Static : never; +export type InputOf = C extends { readonly def: { readonly input: infer I extends Schema } } ? Static : never; /** What a factory returns for contract C; its methods' `ctx.principal` is `P`. */ export type ImplementationOf = C extends RemoteContract diff --git a/packages/chord/src/host.ts b/packages/chord/src/host.ts index 9f96a4909..fe94fd9be 100644 --- a/packages/chord/src/host.ts +++ b/packages/chord/src/host.ts @@ -1136,7 +1136,7 @@ export class HostImpl implements Host, HostCore { return best?.[1]; } - /** Authorize, resolve (maybe spawn), authorize again; returns the synced connection to the host. */ + /** Authorize, resolve (maybe spawn) and admit the host, authorize again; returns the synced connection to the host. */ async #resolve( route: Route, hostName: string, @@ -1148,15 +1148,17 @@ export class HostImpl implements Host, HostCore { if (!(await route.authorize(principal, hostName))) throw refuse(); if (!still()) throw revoked(); const link = await route.resolve(hostName); - if (!still()) throw revoked(); - if (!(await route.authorize(principal, hostName))) throw refuse(); - if (!still()) throw revoked(); + // Admitted before the checks: a host the route spawned for a request that is gone or refused meanwhile still runs, + // and its own references to this host would otherwise wait on a link nobody reads until they time out. let connection = [...this.connections].find((candidate) => candidate.link === link); connection ??= this.#addConnection(link, { role: "downstream", principal: route.principal(hostName), peer: { kind: "host", name: hostName }, }); + if (!still()) throw revoked(); + if (!(await route.authorize(principal, hostName))) throw refuse(); + if (!still()) throw revoked(); await connection.whenSynced(); if (!still()) throw revoked(); return connection; diff --git a/packages/chord/src/next-node/app/README.md b/packages/chord/src/next-node/app/README.md new file mode 100644 index 000000000..254e5da54 --- /dev/null +++ b/packages/chord/src/next-node/app/README.md @@ -0,0 +1,218 @@ +# app + +The app kit (slice 13, design §23 "The app kit", decisions "The app kit, point 1" to "point 5"): an opinionated layer +over `http`, `workers` and `plugins` so an app is its API, its built-in plugins and a start script. `startServer` on the +server, `startWorker` in a worker's entry; the kit makes the data directory, the stores, the hosts, HTTP with deny by +default, sessions and the owner's sign-in, device pairing, each host's files, export and import per address, and the +plugin roots with "apply changes". Node-only. Exported as `@earendil-works/chord/next/app` once v2 has its entry +point. Examples 53 (`53-app/`, a whole small app), 54 (sessions and pairing) and 55 (export and import) run one app. + +## For apps (`index.ts`) + +```ts +// start.ts (the app's start script) +const app = await startServer

({ + name: "acme", // the default data directory: ~/.acme + dataDir: process.argv[2], // optional; no environment variables + root: "./plugins", // the app's built-in plugins (required root); this is the default + listen: { host: "127.0.0.1", port: 8788 }, // 127.0.0.1 and a free port by default + api: { name: "@acme/app", versions: ["v3"], modules: { "@acme/app/v3": v3, "@acme/app/v3/server": v3server } }, + kinds: { server: { platform: "node" }, worker: { platform: "node" } }, + principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", key: address.key }) }, + workers: [{ + kind: Project, // a host kind of the app, any params + entry: "./worker.ts", execArgv: [...], env: {...}, + facet: "worker", // the facet kind its workers load (default) + start: (address, ctx) => ({ id: address.params.id }), // WorkerInfo.start + authorize: (principal, address) => mayReach(principal, address), + idleMs: () => 5 * 60_000, + }], + files: { maxFile: 256 * 2 ** 20, maxTotal: 4 * 2 ** 30 }, // every host's caps (these are the defaults) + install: { npmCommand: ["npm"] }, // how installs run npm + limits: { ephemeralSessions: 1000, pendingPairings: 100, pairingsPerAddress: 10, // the defaults + uploads: 64, importBytes: 4 * 2 ** 30, importEntries: 100_000 }, + web: { entry: "./src/web/main.tsx", page: (request, assets) => shell(request.nonce, assets) }, // slice 11 + tui: { entry: "./src/tui/main.ts" }, + onError: (error) => log(error), // AppError: { kind: "host" | "http" | "workers" | "plugins", error } +}, ctx); +console.log(`Open ${app.ownerLink()}`); +app.host; app.http; app.workers.get("project"); app.plugins.state; app.dataDir; app.ownerTokenFile; +await app.plugins.install({ git: url, ref: "v1" }, ctx); // into /plugins/ +await app.applyChanges(ctx); // build the dirty plugins, roll out: server, then workers +await app.exportData(Project({ id }), "p.tar", ctx); +await app.importData("p.tar", Project({ id: "copy" }), ctx); +await app.close(ctx); // http, workers, host, store, plugins, the lock + +// worker.ts +startWorker

({ api, kinds, busy, onError }); +``` + +- **The principal type** `P` is the app's; the kit needs `principal.server` and `principal.worker(address)` and adds + one variant, its owner: `{ kind: "owner" }` must be one of `P` (a type error otherwise, on `AppOptions`). +- **The API's modules** are named by convention: `/` for every kind and `//` for a + kind's own; a kind's `packages` (React for a web kind) are host packages besides. The modules are this process's + instances; the worker's entry passes the same `api` and `kinds`. +- **`startServer`**, in order: the data directory, by its real path (refused when it is the home directory, a file + system's root, or another user's: the kit would make it private), its lock (one server per data directory, also + through a symbolic link: a second is refused after a second's wait), the layout made private (below), the cookie secret, the server's store, the host (address `Server()`, + `principal.server`), `createHttp` (the links endpoint at `/chord`), the plugins component with the two roots, one + workers component per worker kind (none without worker kinds), the server's plugins loaded with the kit's core + facets (`chord.http`, `chord.app.info`, `chord.app.sessions`, `chord.app.owner`, `chord.app.pairing`, + `chord.app.files`), the owner token file, the listening port, the owner link. A plugin of the required root failing + rejects with the plugins component's `PluginError`, everything closed. `close` is idempotent. +- **`startWorker`** is `workerMain` plus: the worker's directory `workers///` and its store there, the host + at the address acting as `principal.worker(address)`, connected to the server first (trusted, as the server's + principal), its plugins loaded with the worker's core facets (`chord.app.worker-info`, `chord.app.files`), `busy` + from the app. A worker's errors reach the server as a fixed "worker failed" (the error stays in the worker's + `onError`). + +## Start data + +The kit's start-data objects serve process values to plugins: `AppInfo` (`chord.app.info`, on the server: `dataDir`, +`name`, `version`, the newest API version) and `WorkerInfo` (`chord.app.worker-info`, on workers: `address`, `start`, +the start data the app's `start(address)` made). Plugins `self.get(AppInfo, {})` through the app's API, which +re-exports them; nothing else of the process reaches a plugin. + +## Base code + +An app's base code is its API, the start script and the worker's entry; everything else is a plugin. The pattern +(decision "The plugin model", point 8): the providers of the app's core registries and its harness objects are plugins +of the app's required root, not facets the start script passes, so they reload like any plugin and the API only +declares the registries; process values (paths, the start data, the version) reach them through the start-data +objects, not through closures over the start script. Example 53: the API declares `Sections`, the home plugin serves it +and reads `AppInfo`. + +## Sessions and sign-in + +- **Deny by default** (`http`): a route or socket without a principal answers 401 before plugin code runs unless its + contribution says `public: true`; the `/chord` links endpoint needs a principal. +- **Sessions** (`Sessions`, `chord.app.sessions`, local, one per server): `issue({ principal, label, kind: "browser" + | "device" | "ephemeral", secure? })` returns `{ id, token, cookie }`, `verify(token)` the principal or `null`, + `revoke(id)`, `list` (browser and device sessions: `id`, `label`, `kind`, `principal`, `created`, `lastUsed`; never a + token or hash). A token is `.` (32 random bytes, base64url); only the secret's SHA-256 is kept, compared + in constant time. The kit's authenticator reads `Authorization: Bearer ` or the session cookie + `chord-session` (signed with `server/cookie-secret`, `HttpOnly`, `SameSite=Strict`, `Secure` for a remote browser, + 400 days, renewed by a response once older than a day, so a browser used within every 400 days stays signed in; + an ephemeral one only while the browser runs); a malformed token or an id that is no session (a prototype + key included) is a plain 401. Browser and device sessions persist in the server's store; `lastUsed` is written at + most once a minute, and a session unused for 400 days ends (checked when it is used, removed then). Ephemeral + sessions (a public login) live in memory only (`ephemeral-sessions.ts`), never listed: at most + `limits.ephemeralSessions`, none outliving the server's run, so a login costs the same whatever came before. Past + the cap the oldest ends: a flood of logins signs out the earliest ephemeral users (the trade for bounded memory). An + ephemeral session ends with its client's connection (slice 11): once the last Chord link opened with it closed and + none opened for a minute (a reload, a reconnect), or when none opened within a minute of the login; the links + endpoint's `opened` hook counts them. +- **The owner link:** each start makes a one-time token (single use, 10 minutes, the previous start's no longer + valid), `app.ownerLink()` is `http://:/_chord/owner#` (`127.0.0.1` for + `0.0.0.0`, `[::1]` for `::`, an IPv6 address in brackets). The page at `GET /_chord/owner` posts + the fragment's token to `POST /_chord/owner` (public), which consumes it and sets an owner browser session's cookie. + The token stays in the fragment: browsers never send it, so it never reaches a log. No owner by loopback alone. +- **The owner token file** `/owner-token`: a device session of the owner for command-line clients on the same + machine, mode 0600, written in one step (a temporary file made exclusively with mode 0600, then linked or renamed into + place). Kept while its token authenticates as the owner (its session exists, the secret matches, not expired; its + mode set to 0600 again); replaced at the next start otherwise. +- **Device pairing** for clients elsewhere: `POST /_chord/pair` (public, `{ label? }`) returns `{ code, poll }`; the + owner opens `/_chord/pair/` signed in (owner only, a confirm page naming the device) and approves; the + client's `POST /_chord/pair/poll { poll }` answers `{ status: "waiting" }` until then, `{ status: "approved", token }` + once, then 404. Codes are `XXXX-XXXX` from 31 unambiguous characters (`crypto.randomInt`), single use, 10 minutes; + the poll secret is kept hashed. Asking is public, so pairings anonymous clients start are bounded: + `limits.pendingPairings` in all, `limits.pairingsPerAddress` per client address (`HttpRequest.remoteAddress`); past + them 429. The trade: whoever fills the caps locks anonymous pairing out for up to 10 minutes, and behind a tunnel or + relay every remote client shares one address (the tunnel's), so ten requests from one party block remote pairing + for all. A signed-in owner's own `POST /_chord/pair` is not counted: the owner can always start one. Pairings and the owner link live in memory: a restart drops them. + +## Optional plugins + +`defineUsers(profile)` (accounts by id with the app's profile schema: `Users` with `list`, `get`, `update`, +`remove`), `inviteFacet()` (the owner's `POST /_chord/invites { principal, label }` makes a 7-day single-use link, +whose page signs the invitee in as that principal) and `ephemeralLoginFacet

()` (a public `POST /_chord/login { +name }` signing in as `{ kind: "user", id: , name }`, a type error unless `P` allows it). They are exported +facets, not plugin directories: each takes the app's settings as arguments (a profile schema, the principal type), +and a plugin directory could not import the kit (plugins import only the app's API). The app re-exports one through +its API and adds a one-line plugin to its root: + +```ts +// app/v3/server.ts +export const users = defineUsers(obj({ name: str(), avatar: opt(str()) })); +// plugins/users/server.ts +import { users } from "@acme/app/v3/server"; +export default users.facet; +``` + +## Storage and files + +``` +/ mode 0700, like every directory below + server/store.sqlite the server's store, mode 0600 (and its journal files) + server/cookie-secret mode 0600 + server/files/ the server's files, mode 0600 + workers/// a worker's store.sqlite and files/ + plugins/ the user's plugins root + logs/ + owner-token mode 0600 +``` + +Every start sets these modes again, on the server's part when the server starts and on a worker's when it starts: +a data directory made looser before (or restored with other modes) is tightened, so other local users can neither +read a store (it holds the sessions) nor replace it. + +A worker's key is percent-encoded as one directory name (`/`, `\`, `.` and capital letters included, so keys differing +only in case stay apart on a case-insensitive file system), or `sha256-` past 160 characters. + +`Files` (`chord.app.files`, local, one per host: the server's and each worker's) keeps a host's files in its `files/` +and their index (id, name, mime, size, created) in its store: `put({ name, mime, bytes })` (at most 4 MiB), +chunked transfers (design §23): `begin({ name, mime })`, `append({ upload, offset, bytes })` (at most 4 MiB a chunk, +about 1 MiB usual; `offset` must be the bytes stored so far, else `invalid` with the stored `{ offset }`), +`uploaded(upload)` (where a writer resumes after a drop), `finish(upload)`; `read({ id, offset, length })` (at most 4 +MiB), `remove(id)`, `list`. Appends may be sent without waiting for the previous one: each is accepted at the offset +the accepted ones reached and written at its position (one file handle per upload); `uploaded`, `finish` and expiry +wait for the writes in flight, and a failed write ends its upload. Each upload under way holds a file handle, so at +most `limits.uploads` (64) are under way per `Files` object: past it `begin` is refused `unavailable` (`busy`) until +one finishes or an idle one expires. Caps per host (`files` option): a file over `maxFile` or a write past `maxTotal` (uploads +under way count) is refused `too_large`. An upload unwritten for an hour expires; uploads end with the object, and +those a previous run left are dropped when the object goes live. `Files` is `local`: a client uploads through the +plugin's own remote methods, which call it. + +## Export and import + +`exportData(address, path, ctx)` writes the data the worker at `address` created (its store and finished files) to +`path` (mode 0600, staged and renamed), stopping nothing (decision "App kit naming and export"): a running worker packs +its own open store (`HostData`, `data.ts`, a kit object only the server's principal may use; the store's export holds +the commit queue while it reads, so the archive is one consistent point) and its files; with no worker running, the +server takes the worker's process lock (no worker starts meanwhile) and packs the store itself. A file added or removed +while the export runs may be outside that point (the index is the store's): each file is opened when the archive +comes to it (one descriptor at a time), one deleted meanwhile skipped or read from its open handle. `importData` takes the same lock (held: a +worker runs, so the address has data: refused) and changes nothing else. Registering an imported host in the app's own +index is the app's. The archive is a ustar of +`manifest.json` (`{ format: 1, app, api, address }`), `store.archive` (the store's export, JSONL) and `files/…`. +`importData(archive, address, ctx)` refuses an address that has a directory, unpacks beside it and renames into +place: the manifest must come first and name this app and format 1; only regular files and directories under +`manifest.json`, `store.archive` and `files/` are taken; absolute paths, `..`, `.`, empty segments, backslashes, NULs, +links, devices and bad checksums are refused (the kit's own tar reader, `tar.ts`); past `limits.importBytes` unpacked +bytes or `limits.importEntries` entries (counted from the headers before writing) it is refused. A refused or broken +archive leaves nothing. The next hold starts the worker on the imported store and files. + +## Plugins + +Roots: the app's `root` (required: a plugin of it failing stops `startServer`) and `/plugins/` (the user's: +installs go there). `app.plugins.state` is the plugins component's marks for the app's "apply changes" button; +`applyChanges` builds the dirty plugins (and installed ones) and rolls out to the server, then each worker kind; +nothing is applied by itself. + +## Internals + +`app.ts` (`startServer`, the `App`), `worker.ts` (`startWorker`), `layout.ts` (the layout, private directories and +stores), `secrets.ts` (tokens, hashes, files of mode 0600), `sessions.ts`, `owner.ts` (owner link and token file), +`pairing.ts`, `files.ts` and `uploads.ts` (chunked uploads), `info.ts` (start-data objects), `archive.ts` and `tar.ts` +(export and import), `ephemeral-sessions.ts`, and the optional plugins `users.ts`, `invite.ts`, `ephemeral.ts`. + +## Web and terminal (slice 11) + +With `web` the kit builds the page's base bundle from `web.entry`, publishes the `web` kind's plugin scripts and serves +the page through `web.page` at `web.paths` (`["/"]`), signed in unless `web.public` (`webSurface`, +`next-node/surfaces`); with `tui` the terminal's base bundle and the `tui` kind's scripts for launchers +(`tuiSurface`). Plugin assets are served under `/_chord/web/assets/` (the plugins component's `publicPath`). +`applyChanges` rolls out to the server, the workers, then the surfaces (`web`, `tui`: the new set published, clients +not waited for). `app.web` and `app.tui` give the base bundles' hashes. The server also serves the `Themes` registry and +`ThemeChoice` (each principal's dark and light pick, kept in its store). Without `web` and `tui` an app has no client +surface (API only). diff --git a/packages/chord/src/next-node/app/app.ts b/packages/chord/src/next-node/app/app.ts new file mode 100644 index 000000000..5879dd652 --- /dev/null +++ b/packages/chord/src/next-node/app/app.ts @@ -0,0 +1,565 @@ +// `startServer` (slice 13 §1, §5; decisions "The app kit, point 1" to "point 5"): the server of an app built with the kit. +// It lays out the data directory, opens the server's store, creates the server host, the HTTP component and one +// workers component per worker kind, the plugins component over the app's required root and the user's root, loads +// the server's plugins with the kit's own facets (the start-data object, sessions, the owner link, pairing, files), +// and gives the app the controls it needs: the owner link, `applyChanges`, installs, export and import per address. + +import { join, resolve } from "node:path"; +import { type Context, withAbortSignal } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { type Address, type HostKind, hostKind, type KindShape } from "../../next/contracts/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { canonicalJson } from "../../next/data/principal.ts"; +import { createHost, type Facet, type Host, type HostError, type PluginFacet } from "../../next/host/index.ts"; +import { openStore, type Store, sqliteBackend } from "../../next/store/index.ts"; +import { themeChoiceFacet, themesFacet } from "../../next/surfaces/index.ts"; +import { openNodeSqliteDatabase } from "../../sqlite/node.ts"; +import type { Watchable } from "../../watch.ts"; +import { createHttp, type HttpComponent, type HttpError, type ListenAddress } from "../http/index.ts"; +import { + createPlugins, + type HostPackages, + type InstallOptions, + type PluginSource, + type Plugins, + type PluginsError, + type PluginsState, + type RolloutResult, + type RolloutStep, +} from "../plugins/index.ts"; +import { tuiSurface, WEB_PREFIX, type WebSurfaceOptions, webSurface } from "../surfaces/index.ts"; +import { createWorkers, type ProcessLock, processLock, type Workers, type WorkersError } from "../workers/index.ts"; +import { ARCHIVE_FORMAT, type ArchiveManifest, packData, unpackData } from "./archive.ts"; +import { addressParam, HostData } from "./data.ts"; +import { type FilesCaps, filesFacet } from "./files.ts"; +import { appInfoFacet } from "./info.ts"; +import { type AppLayout, createLayout, defaultDataDir, openDataDir, privateStore, workerDirectory } from "./layout.ts"; +import { OWNER_PATH, OwnerLink, type OwnerPrincipal, ownerFacet, ownerTokenFile } from "./owner.ts"; +import { Pairings, pairingFacet } from "./pairing.ts"; +import { secretFile } from "./secrets.ts"; +import { createSessions } from "./sessions.ts"; + +/** The server's host kind: the kit's server host is `Server()`. */ +export const Server = hostKind("server"); + +/** + * The app's API (slice 12): its package name, the versions it provides (newest first) and this process's modules of + * them by specifier (`@acme/app/v3`, `@acme/app/v3/server`). + */ +export type AppApi = { + readonly name: string; + readonly versions: readonly string[]; + readonly modules: HostPackages; +}; + +/** + * A facet kind of the app: where its builds run, and the packages besides the API its plugins share (`react`). Its API + * modules are named by convention: `/` and `//` for every version. + */ +export type AppKind = { readonly platform: "node" | "browser"; readonly packages?: HostPackages }; + +/** A worker kind: a host kind of the app and how its workers start (the workers component's options). */ +export type AppWorkerKind

= { + /** The host kind (`hostKind("project", { id: str() })`). */ + readonly kind: KindShape; + /** The worker's entry module (it calls `startWorker`), its Node flags and environment. */ + readonly entry: string; + readonly execArgv?: readonly string[]; + readonly env?: Readonly>; + /** The facet kind its workers load: `worker` by default. */ + readonly facet?: string; + /** The app's start data for the worker at `address` (JSON), before each start. */ + readonly start: (address: Address, context: Context) => Promise | JsonValue; + /** Whether `principal` may reach the worker at `address` (synchronous). */ + readonly authorize: (principal: P, address: Address) => boolean; + /** How long a worker idles before it stops: 5 minutes by default. */ + readonly idleMs?: () => number; +}; + +/** What the kit reports: from its host, its HTTP component, its workers or its plugins. */ +export type AppError = + | { readonly kind: "host"; readonly error: HostError } + | { readonly kind: "http"; readonly error: HttpError } + | { readonly kind: "workers"; readonly error: WorkersError } + | { readonly kind: "plugins"; readonly error: PluginsError }; + +/** What `startServer` takes. `{ kind: "owner" }` must be one of the app's principals (the kit's owner). */ +export type AppOptions

= { + /** The app's name: its default data directory is `~/.`, its archives carry it. */ + readonly name: string; + /** The data directory (the start script's to decide; no environment variables): `~/.` by default. */ + readonly dataDir?: string; + /** The app's built-in plugins root (required): `plugins/` of the working directory by default. */ + readonly root?: string; + /** Where the server listens: `127.0.0.1`, a free port by default. */ + readonly listen?: ListenAddress; + readonly api: AppApi; + /** The app's facet kinds: `server` and each worker kind's facet kind among them. */ + readonly kinds: Readonly>; + /** The server's principal and each worker's (stamped on its link). */ + readonly principal: { readonly server: P; readonly worker: (address: Address) => P }; + /** The worker kinds; none: a server with clients only. */ + readonly workers?: readonly AppWorkerKind

[]; + /** The caps of each host's files (the server's and every worker's). */ + readonly files?: FilesCaps; + /** The bounds of what public endpoints and imports may cost. */ + readonly limits?: AppLimits; + /** How installs run npm (slice 12's `InstallOptions`): `npm` on the `PATH` by default. */ + readonly install?: InstallOptions; + /** + * The web surface (slice 11): the app's web entry (the base bundle's), its page template, the paths serving the + * page (`["/"]`), whether the page is public; the `web` facet kind. None: no web surface. + */ + readonly web?: AppWeb; + /** The terminal surface (slice 11): the app's terminal entry; the `tui` facet kind. None: no terminal clients. */ + readonly tui?: { readonly entry: string }; + readonly onError: (error: AppError) => void; +} & (OwnerPrincipal extends P ? unknown : { readonly "the app's principal type must have { kind: 'owner' }": never }); + +/** + * The bounds of what public endpoints, clients and imports may cost: live ephemeral sessions (1,000; past it the + * oldest ends, signed out), pairings under way that anonymous clients started (100 in all, 10 per client address; past + * them 429; the owner's own are not counted), uploads under way per `Files` object (64; past it `unavailable`, `busy`), + * an archive's unpacked bytes (4 GiB) and entries (100,000). + */ +export type AppLimits = { + readonly uploads?: number; + readonly ephemeralSessions?: number; + readonly pendingPairings?: number; + readonly pairingsPerAddress?: number; + readonly importBytes?: number; + readonly importEntries?: number; +}; + +/** The kit's web surface options (`webSurface`'s, its kind `web` and plugins the kit's). */ +export type AppWeb = Pick; + +/** The plugins of a running app, for its "apply changes" control and its installs. */ +export type AppPlugins = { + /** The marks (`dirty`, `install`, `set`, `base`): what the app's button shows. */ + readonly state: Watchable; + /** Installs plugins from git or npm into the user's root (slice 12's `installPlugin`); their names. */ + install(source: PluginSource, context: Context): Promise; + /** The plugins component itself. */ + readonly component: Plugins; +}; + +/** A running app. */ +export interface App

{ + readonly host: Host

; + readonly http: HttpComponent

; + /** The workers components, by host kind name. */ + readonly workers: ReadonlyMap>; + readonly plugins: AppPlugins; + /** The web surface's base bundle hash, when the app has one. */ + readonly web: { readonly base: string } | undefined; + /** The terminal surface's base bundle hash, when the app has one. */ + readonly tui: { readonly base: string } | undefined; + /** The data directory. */ + readonly dataDir: string; + /** The owner link of this start (single use, 10 minutes): `http://127.0.0.1:/_chord/owner#`. */ + ownerLink(): string; + /** The owner token file's path (a device session of the owner, mode 0600). */ + readonly ownerTokenFile: string; + /** Builds the dirty plugins and rolls them out: the server, every worker kind, then the clients; never by itself. */ + applyChanges(context: Context): Promise; + /** + * Writes the data the worker at `address` created (its store and finished files) as an archive at `path`, stopping + * nothing: a running worker packs its own store at one consistent point; without one the server packs it under the + * worker's lock (no worker starts meanwhile). + */ + exportData(address: Address, path: string, context: Context): Promise; + /** + * Makes the data of `address` (which must have none yet: nothing is overwritten) from the archive at `archive`, + * under the worker's lock; its next hold starts it. + */ + importData(archive: string, address: Address, context: Context): Promise; + /** Closes the HTTP component, the workers, the host, the store, the watcher and the data directory's lock. */ + close(context: Context): Promise; +} + +/** What `startWorker` reads from its start data besides the app's: the kit's part. */ +export type KitStart = { + readonly dir: string; + readonly facet: string; + readonly principal: JsonValue; + readonly server: JsonValue; + readonly files: { readonly maxFile?: number; readonly maxTotal?: number }; + /** The cap of uploads under way per `Files` object (`limits.uploads`). */ + readonly uploads?: number; +}; + +/** + * Starts the app's server (§1): resolves once its plugins are loaded and it listens. A failing plugin of the app's + * root (or a problem discovery reports there) rejects with the plugins component's `PluginError`, everything closed. + */ +export async function startServer

(options: AppOptions

, context: Context): Promise> { + const { layout, cookieSecret, store, lock } = await openServer( + resolve(options.dataDir ?? defaultDataDir(options.name)), + context, + ); + const { onError, api, limits = {} } = options; + const sessions = createSessions

(cookieSecret, { maxEphemeral: limits.ephemeralSessions }); + const host = createHost

({ + address: Server(), + principal: options.principal.server, + store, + onError: (error) => onError({ kind: "host", error }), + }); + const http = createHttp( + host, + { + listen: options.listen ?? { host: "127.0.0.1", port: 0 }, + links: { path: "/chord", opened: sessions.opened }, + onError: (error) => onError({ kind: "http", error }), + }, + context, + ); + const plugins = createPlugins( + { + roots: [{ path: resolve(options.root ?? "plugins"), required: true }, layout.plugins], + api: { name: api.name, versions: api.versions }, + kinds: Object.fromEntries( + Object.entries(options.kinds).map(([name, kind]) => [ + name, + { platform: kind.platform, hostPackages: hostPackagesOf(api, name, kind) }, + ]), + ), + install: options.install, + publicPath: `${WEB_PREFIX}assets/`, + onError: (error) => onError({ kind: "plugins", error }), + }, + context, + ); + const workers = new Map>(); + for (const kind of options.workers ?? []) { + workers.set(kind.kind.kind, workersOf(host, kind, options, layout.dataDir, plugins, context)); + } + const link = new OwnerLink(); + const core: PluginFacet

[] = [ + { name: "chord.http", facet: http.facet }, + { + name: "chord.app.info", + facet: appInfoFacet({ dataDir: layout.dataDir, name: options.name, version: api.versions[0] ?? "" }), + }, + { name: "chord.app.sessions", facet: sessions.facet }, + { name: "chord.app.owner", facet: ownerFacet(link) }, + { + name: "chord.app.pairing", + facet: pairingFacet( + new Pairings({ maxPending: limits.pendingPairings, perAddress: limits.pairingsPerAddress }), + ), + }, + { name: "chord.app.files", facet: filesFacet(join(layout.server, "files"), options.files, limits.uploads) }, + { name: "chord.themes", facet: themesFacet() }, + { name: "chord.themes.choice", facet: themeChoiceFacet() }, + ]; + const surfaces: Surface

[] = []; + const app = new RunningApp(host, http, workers, plugins, surfaces, { store, lock }, layout, options); + try { + for (const surface of await surfacesOf(host, options, plugins, context)) { + surfaces.push(surface); + core.push({ name: `chord.${surface.name}`, facet: surface.facet }); + } + await plugins.load(host, { kind: "server", core, modules: modulesOf(api, options.kinds.server) }, context); + await ownerTokenFile(layout.ownerToken, host, context); + app.started(link.renew(), await http.listening(context)); + } catch (error) { + await app.close(context); + throw error; + } + return app; +} + +/** A client surface of the app: its name (`web`, `tui`), its facet, its rollout step, its base bundle's hash. */ +type Surface

= { + readonly name: "web" | "tui"; + readonly facet: Facet

; + step(name: string): RolloutStep; + readonly base: string; +}; + +/** The app's client surfaces, built (their base bundles) in order: the web's, the terminal's. */ +async function surfacesOf

( + host: Host

, + options: AppOptions

, + plugins: Plugins, + context: Context, +): Promise[]> { + const surfaces: Surface

[] = []; + if (options.web !== undefined) + surfaces.push({ name: "web", ...(await webSurface(host, { ...options.web, plugins }, context)) }); + if (options.tui !== undefined) { + surfaces.push({ name: "tui", ...(await tuiSurface(host, { entry: options.tui.entry, plugins }, context)) }); + } + return surfaces; +} + +/** How long a second server waits for the data directory's lock before it is refused. */ +const SERVER_LOCK_MS = 1000; + +/** + * The server's own part of the data directory `path`: its lock (one server per data directory, by its real path: a + * second one, also through a symbolic link, would share its stores and secrets), the layout made private, the cookie + * secret and the server's store; the lock released when the rest fails. + */ +async function openServer( + path: string, + context: Context, +): Promise<{ layout: AppLayout; cookieSecret: string; store: Store; lock: ProcessLock }> { + const dataDir = await openDataDir(path); + const waited = AbortSignal.timeout(SERVER_LOCK_MS); + const lock = await processLock(`chord-app:${dataDir}`, withAbortSignal(waited, context)).catch((error: unknown) => { + throw waited.aborted ? new Error(`startServer: another server runs on ${dataDir}`) : error; + }); + try { + const layout = await createLayout(dataDir); + const cookieSecret = await secretFile(layout.cookieSecret); + await privateStore(layout.serverStore); + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(layout.serverStore)), context); + return { layout, cookieSecret, store, lock }; + } catch (error) { + await lock.release(); + throw error; + } +} + +/** The host packages of kind `name`: the API's shared and per-kind module of every version, and its packages. */ +function hostPackagesOf(api: AppApi, name: string, kind: AppKind): string[] { + const modules = api.versions.flatMap((version) => [`${api.name}/${version}`, `${api.name}/${version}/${name}`]); + return [...modules, ...Object.keys(kind.packages ?? {})]; +} + +/** This process's modules for a kind: the API's and the kind's packages. */ +function modulesOf(api: AppApi, kind: AppKind | undefined): HostPackages { + return { ...api.modules, ...kind?.packages }; +} + +/** The workers component of one worker kind: started with the kit's part of the start data beside the app's. */ +function workersOf

( + host: Host

, + kind: AppWorkerKind

, + options: AppOptions

, + dataDir: string, + plugins: Plugins, + context: Context, +): Workers { + const facet = kind.facet ?? "worker"; + const serverKey = canonicalJson(options.principal.server as unknown as JsonValue, context); + const isServer = (principal: P): boolean => canonicalJson(principal as unknown as JsonValue, context) === serverKey; + return createWorkers( + host, + // A host kind of the app: any params; its addresses come back from its route as the kit hands them on. + kind.kind as HostKind, + { + kind: facet, + entry: kind.entry, + ...(kind.execArgv === undefined ? {} : { execArgv: kind.execArgv }), + ...(kind.env === undefined ? {} : { env: kind.env }), + // The server reaches its workers' kit objects (`HostData`) whatever the app admits. + authorize: (principal, address) => isServer(principal) || kind.authorize(principal, address), + principal: options.principal.worker, + start: async (address, ctx) => { + const kit: KitStart = { + dir: workerDirectory(dataDir, address), + facet, + // Principals are JSON (they cross links). + principal: options.principal.worker(address) as unknown as JsonValue, + server: options.principal.server as unknown as JsonValue, + files: { ...options.files }, + ...(options.limits?.uploads === undefined ? {} : { uploads: options.limits.uploads }), + }; + return { kit, app: await kind.start(address, ctx) }; + }, + lock: (address) => workerLock(options.name, dataDir, address), + ...(kind.idleMs === undefined ? {} : { idleMs: kind.idleMs }), + plugins, + onError: (error) => options.onError({ kind: "workers", error }), + }, + context, + ); +} + +/** The process lock of the worker at `address`: its worker holds it while it runs, an export or import meanwhile. */ +function workerLock(name: string, dataDir: string, address: Address): string { + return `${name}:${dataDir}:${address.key}`; +} + +/** How long an export or import waits for a worker's lock: past it a worker holds it (it runs or starts). */ +const ADDRESS_LOCK_MS = 2000; + +/** The host of the owner link for the address listened on: loopback for a wildcard, an IPv6 address in brackets. */ +function linkHost(host: string): string { + if (host === "0.0.0.0" || host === "") return "127.0.0.1"; + if (host === "::") return "[::1]"; + return host.includes(":") ? `[${host}]` : host; +} + +class RunningApp

implements App

{ + readonly host: Host

; + readonly http: HttpComponent

; + readonly workers: ReadonlyMap>; + readonly plugins: AppPlugins; + readonly dataDir: string; + readonly ownerTokenFile: string; + readonly #component: Plugins; + /** The client surfaces, filled before the server's load. */ + readonly #surfaces: readonly Surface

[]; + readonly #store: Store; + readonly #lock: ProcessLock; + readonly #options: AppOptions

; + #link = ""; + #closing: Promise | undefined; + + constructor( + host: Host

, + http: HttpComponent

, + workers: ReadonlyMap>, + plugins: Plugins, + surfaces: readonly Surface

[], + server: { readonly store: Store; readonly lock: ProcessLock }, + layout: { readonly dataDir: string; readonly plugins: string; readonly ownerToken: string }, + options: AppOptions

, + ) { + this.host = host; + this.http = http; + this.workers = workers; + this.#component = plugins; + this.#surfaces = surfaces; + this.#store = server.store; + this.#lock = server.lock; + this.#options = options; + this.dataDir = layout.dataDir; + this.ownerTokenFile = layout.ownerToken; + this.plugins = { + state: plugins.state, + install: (source, context) => plugins.installPlugin(source, layout.plugins, context), + component: plugins, + }; + } + + /** The start finished: the owner link's token and the address it names (the one listened on). */ + started(token: string, listening: ListenAddress): void { + this.#link = `http://${linkHost(listening.host)}:${listening.port}${OWNER_PATH}#${token}`; + } + + ownerLink(): string { + return this.#link; + } + + get web(): { readonly base: string } | undefined { + return this.#surfaces.find(({ name }) => name === "web"); + } + + get tui(): { readonly base: string } | undefined { + return this.#surfaces.find(({ name }) => name === "tui"); + } + + async applyChanges(context: Context): Promise { + const plugins = this.#component; + const built = await plugins.build(plugins.state.value.dirty, context); + const steps = [ + plugins.step("server", this.host), + ...[...this.workers].map(([name, each]) => each.step(name)), + ...this.#surfaces.map((surface) => surface.step(surface.name)), + ]; + return plugins.rollout(steps, built, context); + } + + async exportData(address: Address, path: string, context: Context): Promise { + const workers = this.#workersOf(address); + const manifest = this.#manifest(address); + const target = resolve(path); + // Running: the worker packs its own store. Else the lock, unless a worker starts first: then ask it. + for (let attempt = 1; ; attempt++) { + if (workers.state.value[address.key]?.status === "running") { + const data = await this.host.get(HostData, { address: addressParam(address) }, context); + try { + await data.exportTo({ path: target, manifest: JSON.stringify(manifest) }, context); + } finally { + data.release(); + } + return; + } + const lock = await this.#workerLock(address, context); + if (lock !== undefined) { + try { + await packData(workerDirectory(this.dataDir, address), manifest, target, context); + } finally { + await lock.release(); + } + return; + } + if (attempt === 5) throw new Error(`exportData: the worker at ${address.key} keeps starting; try again`); + } + } + + async importData(archive: string, address: Address, context: Context): Promise { + this.#workersOf(address); + const { name, limits = {} } = this.#options; + const accept = (manifest: ArchiveManifest): void => { + // A trust boundary: an archive from anywhere. + if (manifest?.format !== ARCHIVE_FORMAT) throw new Error(`importData: not a format ${ARCHIVE_FORMAT} archive`); + if (manifest.app !== name) throw new Error(`importData: an archive of "${manifest.app}", not "${name}"`); + }; + const lock = await this.#workerLock(address, context); + // A worker holds it: it runs, so the address has data, and nothing is overwritten. + if (lock === undefined) throw new Error(`importData: ${address.key} has data (its worker runs)`); + try { + const bounds = { bytes: limits.importBytes, entries: limits.importEntries }; + await unpackData(resolve(archive), workerDirectory(this.dataDir, address), accept, bounds, context); + } finally { + await lock.release(); + } + } + + /** The process lock of the worker at `address` (the one its worker takes at start), or `undefined` while held. */ + async #workerLock(address: Address, context: Context): Promise { + const waited = AbortSignal.timeout(ADDRESS_LOCK_MS); + try { + return await processLock( + workerLock(this.#options.name, this.dataDir, address), + withAbortSignal(waited, context), + ); + } catch (error) { + if (waited.aborted) return undefined; + throw error; + } + } + + close(context: Context): Promise { + this.#closing ??= this.#close(context); + return this.#closing; + } + + /** The workers of `address`'s kind. */ + #workersOf(address: Address): Workers { + const workers = this.workers.get(address.kind); + if (workers === undefined) throw new Error(`app: no worker kind "${address.kind}"`); + return workers; + } + + /** The manifest of `address`'s archive. */ + #manifest(address: Address): ArchiveManifest { + const { name, api } = this.#options; + return { + format: ARCHIVE_FORMAT, + app: name, + api: api.versions[0] ?? "", + address: { kind: address.kind, key: address.key }, + }; + } + + /** + * The close, in order: HTTP first (it drains requests the host's close would seal), workers, host, store, the + * plugins' watcher, then the data directory's lock. + */ + async #close(context: Context): Promise { + await this.http.close(context); + for (const workers of this.workers.values()) await workers.close(context); + await this.host.close(context); + await this.#store.close(context); + this.#component.close(); + await this.#lock.release(); + } +} diff --git a/packages/chord/src/next-node/app/archive.ts b/packages/chord/src/next-node/app/archive.ts new file mode 100644 index 000000000..a0b420f1d --- /dev/null +++ b/packages/chord/src/next-node/app/archive.ts @@ -0,0 +1,229 @@ +// Data archives (slice 13 §4, decisions "The app kit, point 5" and "App kit naming and export"): one tar of the data a +// host created, `manifest.json` (the app, its API version, the kit's format version, the address), `store.archive` (the +// store's export archive, slice 6a, as JSONL: one consistent point, the export holding the commit queue while it reads) +// and `files/` (its finished files). A running worker packs its own open store; without one the server packs the +// directory under the worker's lock. Unpacking makes a new directory for an address that has none, refusing every path +// that would leave it. + +import { createReadStream, createWriteStream, type Dirent } from "node:fs"; +import { mkdir, mkdtemp, open, readdir, rename, rm, stat } from "node:fs/promises"; +import { dirname, join, relative, sep } from "node:path"; +import { pipeline } from "node:stream/promises"; +import type { Context } from "../../context/index.ts"; +import { decodeJsonl, encodeJsonl, importStore, openStore, type Store, sqliteBackend } from "../../next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../sqlite/node.ts"; +import { privateDirectory, privateStore } from "./layout.ts"; +import { readTar, type TarEntry, writeTar } from "./tar.ts"; + +/** The kit's archive format version. */ +export const ARCHIVE_FORMAT = 1; + +/** What an archive says about itself. */ +export type ArchiveManifest = { + readonly format: number; + readonly app: string; + readonly api: string; + readonly address: { readonly kind: string; readonly key: string }; +}; + +/** The store file of a host directory. */ +const STORE = "store.sqlite"; + +/** + * Packs the host directory `dir` of a host that does not run (its worker's lock held: nothing writes it) into the + * archive at `path`: `manifest`, its store's export and its files. + */ +export async function packData(dir: string, manifest: ArchiveManifest, path: string, context: Context): Promise { + if (!(await exists(dir))) throw new Error(`exportData: ${manifest.address.key} has no data`); + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, STORE))), context); + try { + await packStore(store, join(dir, "files"), manifest, path, context); + } finally { + await store.close(context); + } +} + +/** + * Packs the open `store` and the files directory `files` into the archive at `path` (mode 0600, staged beside it and + * renamed): `manifest`, the store's export (one consistent point, written while the host runs on) and the finished + * files (uploads under way left out). + */ +export async function packStore( + store: Store, + files: string, + manifest: ArchiveManifest, + path: string, + context: Context, +): Promise { + const staging = await mkdtemp(join(dirname(path), ".chord-export-")); + try { + const exported = join(staging, "store.archive"); + await pipeline(encodeJsonl(store.export(context)), createWriteStream(exported, { mode: 0o600 })); + const text = new TextEncoder().encode(JSON.stringify(manifest)); + const size = (await stat(exported)).size; + async function* entries(): AsyncGenerator { + yield { path: "manifest.json", size: text.length, data: chunksOf(text) }; + yield { path: "store.archive", size, data: createReadStream(exported) }; + yield* filesOf(files); + } + const written = join(staging, "archive.tar"); + await pipeline(writeTar(entries()), createWriteStream(written, { mode: 0o600 })); + await rename(written, path); + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + +/** One chunk. */ +async function* chunksOf(bytes: Uint8Array): AsyncGenerator { + yield bytes; +} + +/** + * The files under `root` as entries `files/` (hidden ones, uploads under way, left out), each opened only + * when the archive comes to it (one descriptor at a time) and its size taken from the open file: a file deleted while a + * running worker is exported is skipped, or read whole from its open handle (finished files never change; Opus L7). + */ +async function* filesOf(root: string): AsyncGenerator { + for (const path of await pathsUnder(root)) { + const handle = await open(path, "r").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); + if (handle === undefined) continue; + const info = await handle.stat(); + if (!info.isFile()) { + await handle.close(); + continue; + } + const inside = relative(root, path).split(sep).join("/"); + yield { + path: `files/${inside}`, + size: info.size, + data: handle.createReadStream({ start: 0, end: Math.max(0, info.size - 1) }), + }; + } +} + +/** The paths of the files under `root`, sorted, hidden ones left out (a directory gone meanwhile: none of it). */ +async function pathsUnder(root: string): Promise { + const paths: string[] = []; + const walk = async (dir: string): Promise => { + let entries: Dirent[]; + try { + entries = (await readdir(dir, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : 1)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + for (const entry of entries) { + if (entry.name.startsWith(".")) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile()) paths.push(path); + } + }; + await walk(root); + return paths; +} + +/** What an import takes at most: unpacked bytes (4 GiB by default) and entries (100,000 by default). */ +export type ImportLimits = { readonly bytes?: number; readonly entries?: number }; + +/** + * Unpacks the archive at `archive` into the new host directory `dir` (refused when it exists): its manifest checked by + * `accept` (a throw refuses it), the store made from `store.archive`, `files/` restored, within `limits`. Staged beside + * `dir` (mode 0700, files 0600) and renamed into place at the end, so a refused or broken archive leaves nothing. + */ +export async function unpackData( + archive: string, + dir: string, + accept: (manifest: ArchiveManifest) => void, + limits: ImportLimits, + context: Context, +): Promise { + if (await exists(dir)) throw new Error(`importData: ${dir} exists already (an address is imported once)`); + await privateDirectory(dirname(dir)); + const staging = await mkdtemp(join(dirname(dir), ".chord-import-")); + try { + const storeArchive = await unpackEntries(archive, staging, accept, limits, context); + await privateStore(join(staging, STORE)); + const backend = sqliteBackend(await openNodeSqliteDatabase(join(staging, STORE))); + const store = await importStore(backend, decodeJsonl(createReadStream(storeArchive)), context); + await store.close(context); + await rm(storeArchive); + await rename(staging, dir); + } catch (error) { + await rm(staging, { recursive: true, force: true }); + throw error; + } +} + +/** + * Writes the entries of `archive` into `staging`: the manifest first (checked by `accept` before any data is written), + * then `store.archive` and `files/…`, within `limits` (counted from the headers, before an entry is written). The path + * of the staged `store.archive`. + */ +async function unpackEntries( + archive: string, + staging: string, + accept: (manifest: ArchiveManifest) => void, + limits: ImportLimits, + context: Context, +): Promise { + const maxBytes = limits.bytes ?? 4 * 1024 ** 3; + const maxEntries = limits.entries ?? 100_000; + let manifest: ArchiveManifest | undefined; + let storeArchive: string | undefined; + let bytes = 0; + let entries = 0; + for await (const entry of readTar(createReadStream(archive))) { + context.abortSignal?.throwIfAborted(); + if (++entries > maxEntries) throw new Error(`importData: the archive has more than ${maxEntries} entries`); + if (entry.kind === "directory") continue; + bytes += entry.size; + if (bytes > maxBytes) throw new Error(`importData: the archive unpacks to more than ${maxBytes} bytes`); + if (manifest === undefined && entry.path !== "manifest.json") { + throw new Error("importData: the archive does not start with manifest.json"); + } + if (entry.path === "manifest.json") { + // A trust boundary: the archive's own description, checked by `accept`. + manifest = JSON.parse(new TextDecoder().decode(await collect(entry.data, 64 * 1024))) as ArchiveManifest; + accept(manifest); + } else if (entry.path === "store.archive") { + storeArchive = join(staging, "store.archive"); + await pipeline(entry.data, createWriteStream(storeArchive, { mode: 0o600 })); + } else if (entry.path.startsWith("files/")) { + // `readTar` refused absolute paths and `..`: `files/` stays inside the staging directory. + const target = join(staging, ...entry.path.split("/")); + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); + await pipeline(entry.data, createWriteStream(target, { mode: 0o600 })); + } else { + throw new Error(`importData: the archive holds ${entry.path}, which an address archive does not`); + } + } + if (manifest === undefined || storeArchive === undefined) { + throw new Error("importData: the archive has no manifest.json or store.archive"); + } + return storeArchive; +} + +/** The bytes of `data`, refused past `limit`. */ +async function collect(data: AsyncIterable, limit: number): Promise { + const chunks: Uint8Array[] = []; + let size = 0; + for await (const chunk of data) { + size += chunk.length; + if (size > limit) throw new Error("importData: manifest.json is too large"); + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + +/** Whether `path` exists. */ +async function exists(path: string): Promise { + return stat(path).then( + () => true, + () => false, + ); +} diff --git a/packages/chord/src/next-node/app/data.ts b/packages/chord/src/next-node/app/data.ts new file mode 100644 index 000000000..df18e6dce --- /dev/null +++ b/packages/chord/src/next-node/app/data.ts @@ -0,0 +1,52 @@ +// A host's data on its worker (decision "App kit naming and export"): the kit's object a running worker serves, through +// which the server exports the worker's data without stopping it. The worker packs its own open store (its export holds +// the commit queue while it reads: one consistent point) and its finished files into the archive at the path the server +// names. Only the server may ask (its principal); the worker's plugins never see it. + +import { BACKGROUND_CONTEXT } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { type Address, method, obj, remote, str } from "../../next/contracts/index.ts"; +import { ChordError, type Principal } from "../../next/data/index.ts"; +import { canonicalJson } from "../../next/data/principal.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import type { Store } from "../../next/store/index.ts"; +import { type ArchiveManifest, packStore } from "./archive.ts"; + +/** The data of the worker at `address` (its JSON, as `addressParam` writes it): `exportTo` packs it at `path`. */ +export const HostData = remote("chord.app.data", { + version: 1, + params: { address: str() }, + // The address a worker has: the server names it (`addressParam`); only its kind and params place the object. + hostAddress: ({ address }) => JSON.parse(address) as Address, + variants: { main: { methods: { exportTo: method(obj({ path: str(), manifest: str() })) } } }, +}); + +/** `address` as `HostData`'s param. */ +export function addressParam(address: Address): string { + return JSON.stringify({ kind: address.kind, params: address.params, key: address.key }); +} + +/** + * The kit's worker facet serving `HostData` for the worker's open `store` and its files directory `files`, to `server` + * (the server's principal) only. + */ +export function hostDataFacet

(store: Store, files: string, server: P): Facet

{ + const serverKey = canonicalJson(server as JsonValue, BACKGROUND_CONTEXT); + return defineFacet

((host) => + host.serve(HostData, (_params, principal) => { + if (canonicalJson(principal as JsonValue, BACKGROUND_CONTEXT) !== serverKey) { + throw new ChordError({ code: "forbidden", message: "chord.app.data: the server's only" }); + } + return { + variant: "main", + members: { + exportTo: async ({ path, manifest }, ctx) => { + // The server's own manifest for this address. + await packStore(store, files, JSON.parse(manifest) as ArchiveManifest, path, ctx); + return null; + }, + }, + }; + }), + ); +} diff --git a/packages/chord/src/next-node/app/ephemeral-sessions.ts b/packages/chord/src/next-node/app/ephemeral-sessions.ts new file mode 100644 index 000000000..a64f6404a --- /dev/null +++ b/packages/chord/src/next-node/app/ephemeral-sessions.ts @@ -0,0 +1,80 @@ +// Ephemeral sessions (slice 13 §3, carry-over to slice 11): a pick-a-name login's sessions, in memory only, at most +// `max` (past it the oldest ends), and each ending with its client's connection: once the last Chord link +// authenticated with it closed and none opened for a grace period (a page reload, a reconnect after a drop), or when +// none opened within that period of the login. + +import type { Principal } from "../../next/data/index.ts"; + +/** How long an ephemeral session lives without a link: a minute, past a reload and the reconnect loop's pauses. */ +export const EPHEMERAL_GRACE_MS = 60_000; + +/** One ephemeral session: its secret's hash, its principal, its open links, its end's timer while it has none. */ +type Session = { + readonly hash: string; + readonly principal: Principal; + links: number; + timer: ReturnType | undefined; +}; + +/** The server's ephemeral sessions. */ +export class EphemeralSessions { + readonly #max: number; + readonly #graceMs: number; + /** Insertion order is age order: the first entry is the oldest. */ + readonly #sessions = new Map(); + + constructor(max: number, graceMs = EPHEMERAL_GRACE_MS) { + this.#max = max; + this.#graceMs = graceMs; + } + + /** A new session `id` (its secret's hash `hash`) for `principal`; the oldest ends past the cap. */ + issue(id: string, hash: string, principal: Principal): void { + const session: Session = { hash, principal, links: 0, timer: undefined }; + this.#sessions.set(id, session); + this.#waitForLink(id, session); + for (const [oldest] of this.#sessions) { + if (this.#sessions.size <= this.#max) break; + this.revoke(oldest); + } + } + + /** Session `id`'s hash and principal, while it lives. */ + get(id: string): { readonly hash: string; readonly principal: Principal } | undefined { + return this.#sessions.get(id); + } + + /** Ends session `id`; whether it was one. */ + revoke(id: string): boolean { + const session = this.#sessions.get(id); + if (session === undefined) return false; + clearTimeout(session.timer); + this.#sessions.delete(id); + return true; + } + + /** A link authenticated with session `id` opened: what to call when it closes; `undefined` for no such session. */ + linkOpened(id: string): (() => void) | undefined { + const session = this.#sessions.get(id); + if (session === undefined) return undefined; + session.links++; + clearTimeout(session.timer); + session.timer = undefined; + let closed = false; + return () => { + if (closed) return; + closed = true; + session.links--; + if (session.links === 0 && this.#sessions.get(id) === session) this.#waitForLink(id, session); + }; + } + + /** Ends `session` unless a link opens within the grace period. */ + #waitForLink(id: string, session: Session): void { + session.timer = setTimeout(() => { + if (session.links === 0) this.revoke(id); + }, this.#graceMs); + // The server's process does not wait for it. + session.timer.unref?.(); + } +} diff --git a/packages/chord/src/next-node/app/ephemeral.ts b/packages/chord/src/next-node/app/ephemeral.ts new file mode 100644 index 000000000..4d162dbdb --- /dev/null +++ b/packages/chord/src/next-node/app/ephemeral.ts @@ -0,0 +1,43 @@ +// The ephemeral login plugin (slice 13 §3, decision "The app kit, point 2"): an IRC-like "pick a name and you're in". +// `POST /_chord/login` (public: `{ name }`) issues an ephemeral session for `{ kind: "user", id: , name }` +// (the app's principal type must allow it) and answers with its cookie. No account is stored. The session ends with +// the browser (its cookie has no lifetime) or the server (ephemeral sessions are dropped whenever it starts); ending it +// with the client's last link waits for slice 11's clients (carry-over). An exported facet, as `defineUsers`. + +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import { Http } from "../http/index.ts"; +import { newToken } from "./secrets.ts"; +import { Sessions } from "./sessions.ts"; + +/** The principal an ephemeral login gives. */ +export type EphemeralUser = { readonly kind: "user"; readonly id: string; readonly name: string }; + +/** The longest name taken. */ +const MAX_NAME = 40; + +/** The ephemeral login plugin's server facet; a type error unless the app's principal type holds `EphemeralUser`. */ +export function ephemeralLoginFacet

( + ..._fits: EphemeralUser extends P ? [] : ["the app's principal type must hold { kind: 'user', id, name }"] +): Facet

{ + return defineFacet

((_host, self) => + self.contribute(Http, { + routes: { + "POST /_chord/login": { + public: true, + handler: async (request, ctx) => { + // A trust boundary: anyone may log in under a name. + const body = (await request.json(4096)) as { readonly name?: unknown } | null; + const name = typeof body?.name === "string" ? body.name.trim().slice(0, MAX_NAME) : ""; + if (name === "") return { kind: "error", status: 400 }; + const principal: EphemeralUser = { kind: "user", id: newToken(12), name }; + const sessions = await ctx.get(Sessions, {}); + const secure = request.provenance.kind !== "local"; + const issued = await sessions.issue({ principal, label: name, kind: "ephemeral", secure }, ctx); + return { kind: "json", value: principal, headers: { "Set-Cookie": issued.cookie } }; + }, + }, + }, + }), + ); +} diff --git a/packages/chord/src/next-node/app/files.ts b/packages/chord/src/next-node/app/files.ts new file mode 100644 index 000000000..35abda7e3 --- /dev/null +++ b/packages/chord/src/next-node/app/files.ts @@ -0,0 +1,196 @@ +// Files (slice 13 §4, decision "The app kit, point 5: plugins, files and export"): the kit's file storage, one `Files` +// object per host (`self.get(Files, {})`), its files in the host's directory (`files/`) and its index (id, name, mime, +// size, created) a watchable member kept in the store. Small files in one call (`put`), large ones in chunks (design +// §23 "Chunked transfers": `begin`, `append` at the stored offset, `uploaded` to resume after a drop, `finish`; +// unfinished uploads expire), reads by range, caps per host. Plugins never see a path. + +import { mkdir, open, readdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { arr, bytes, int, local, map, method, num, obj, str } from "../../next/contracts/index.ts"; +import { ChordError, type Principal } from "../../next/data/index.ts"; +import { invalid } from "../../next/data/internal.ts"; +import { defineState } from "../../next/documents/index.ts"; +import { type Ctx, defineFacet, type Facet } from "../../next/host/index.ts"; +import { newToken } from "./secrets.ts"; +import { Uploads } from "./uploads.ts"; + +/** A file as the index lists it. */ +export type FileInfo = { + readonly id: string; + readonly name: string; + readonly mime: string; + readonly size: number; + readonly created: number; +}; + +/** The caps of one host's files (bytes). */ +export type FilesCaps = { + /** The largest file: 256 MiB by default. */ + readonly maxFile?: number; + /** All files together: 4 GiB by default. */ + readonly maxTotal?: number; +}; + +/** One file's fields in the index. */ +const Entry = obj({ name: str(), mime: str(), size: int(), created: num() }); + +/** The host's files (one per host): the index, small and chunked writes, ranged reads, removal. */ +export const Files = local("chord.app.files", { + version: 1, + params: {}, + members: { + list: arr(obj({ id: str(), name: str(), mime: str(), size: int(), created: num() })), + /** Stores a small file (at most 4 MiB) in one call; its id. */ + put: method(obj({ name: str(), mime: str(), bytes: bytes() }), str()), + /** Starts a chunked upload; its upload id. `unavailable` (`busy`) while the host's cap of uploads are under way. */ + begin: method(obj({ name: str(), mime: str() }), str()), + /** + * Appends a chunk (at most 4 MiB; about 1 MiB is the usual size) to upload `upload` at `offset`, which must be + * the bytes stored so far (else `invalid`, its details `{ offset }` the stored one). + */ + append: method(obj({ upload: str(), offset: int({ min: 0 }), bytes: bytes() })), + /** The bytes stored so far of upload `upload`: where a writer resumes after a drop. */ + uploaded: method(str(), int()), + /** Ends upload `upload`: the file it made, its id. */ + finish: method(str(), str()), + /** `length` bytes of file `id` from `offset` (fewer at its end). */ + read: method(obj({ id: str(), offset: int({ min: 0 }), length: int({ min: 0 }) }), bytes()), + /** Removes file `id` (an unknown one: nothing). */ + remove: method(str()), + }, +}); + +/** The index document. */ +const FilesIndex = defineState("chord.app.files", { + schema: obj({ files: map(Entry) }), + version: 1, + initial: () => ({ files: {} }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 50, +}); + +/** The largest piece a call carries (a `put`, an `append`, a `read`): 4 MiB, well under a link's part limit. */ +const MAX_PIECE = 4 * 1024 * 1024; + +/** How long an upload may go unwritten before it expires: an hour. */ +const UPLOAD_IDLE_MS = 60 * 60_000; + +/** How often expired uploads are dropped: every minute. */ +const SWEEP_MS = 60_000; + +/** + * The kit's facet serving `Files` with the host's files directory `dir` (`/files`), `caps` and at most + * `uploads` uploads under way (64 by default). Uploads under way live in `dir/.uploads//` (`Uploads`) and end with + * the object (an unfinished one is dropped when the next one goes live). + */ +export function filesFacet

(dir: string, caps: FilesCaps = {}, uploads = 64): Facet

{ + const maxFile = caps.maxFile ?? 256 * 1024 * 1024; + const maxTotal = caps.maxTotal ?? 4 * 1024 * 1024 * 1024; + const uploadsDir = join(dir, ".uploads"); + return defineFacet

((host) => + host.serve(Files, (_params, self) => { + const index = self.state(FilesIndex); + // This run's uploads in their own directory: an upload left by an earlier run (its process ended) can never + // finish and is dropped, while this run's may already have begun. + const run = join(uploadsDir, newToken(12)); + const underway = new Uploads(run, uploads); + // Uploads under way count against the total too: two of them cannot pass it together. + const room = (adding: number, fileSize: number): void => { + if (fileSize > maxFile) + throw tooLarge(`a file of ${fileSize} bytes is larger than this host's cap (${maxFile})`); + const stored = Object.values(index.value.files).reduce((sum, file) => sum + file.size, 0); + if (stored + underway.size + adding > maxTotal) { + throw tooLarge(`this host's files would exceed their cap (${maxTotal} bytes)`); + } + }; + self.onLive((ctx) => sweep(uploadsDir, run, underway, ctx)); + const fileOf = (id: string): FileInfo | undefined => + // A trust boundary: `id` is the caller's (`Object.hasOwn`: no inherited member is a file). + Object.hasOwn(index.value.files, id) ? { id, ...index.value.files[id]! } : undefined; + return { + list: self.derive([index], ([value]) => + Object.entries(value.files).map(([id, { name, mime, size, created }]) => ({ + id, + name, + mime, + size, + created, + })), + ), + put: async ({ name, mime, bytes }, ctx) => { + if (bytes.length > MAX_PIECE) + throw tooLarge(`put takes at most ${MAX_PIECE} bytes: use begin, append, finish`); + room(bytes.length, bytes.length); + const id = newToken(12); + await mkdir(dir, { recursive: true, mode: 0o700 }); + await writeFile(join(dir, id), bytes, { mode: 0o600 }); + await index.change((draft) => { + draft.files[id] = { name, mime, size: bytes.length, created: Date.now() }; + }, ctx); + return id; + }, + begin: ({ name, mime }) => underway.begin(name, mime), + append: async ({ upload, offset, bytes }) => { + if (bytes.length > MAX_PIECE) throw tooLarge(`append takes at most ${MAX_PIECE} bytes`); + await underway.append(upload, offset, bytes, room); + return null; + }, + uploaded: (upload) => underway.uploaded(upload), + finish: async (upload, ctx) => { + const id = newToken(12); + await mkdir(dir, { recursive: true, mode: 0o700 }); + const { name, mime, size } = await underway.finish(upload, join(dir, id)); + await index.change((draft) => { + draft.files[id] = { name, mime, size, created: Date.now() }; + }, ctx); + return id; + }, + read: async ({ id, offset, length }) => { + const file = fileOf(id); + if (file === undefined) throw invalid("args", `files: no file ${id}`); + if (length > MAX_PIECE) throw tooLarge(`read takes at most ${MAX_PIECE} bytes`); + const handle = await open(join(dir, id), "r"); + try { + const buffer = new Uint8Array(Math.max(0, Math.min(length, file.size - offset))); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, offset); + return buffer.subarray(0, bytesRead); + } finally { + await handle.close(); + } + }, + remove: async (id, ctx) => { + if (fileOf(id) === undefined) return null; + await index.change((draft) => { + delete draft.files[id]; + }, ctx); + await rm(join(dir, id), { force: true }); + return null; + }, + }; + }), + ); +} + +/** + * The files object's routine once live: drops the uploads earlier runs left (every directory under `uploadsDir` but + * `run`), then every minute the uploads unwritten for an hour; when the object ends, every upload still under way. + */ +async function sweep(uploadsDir: string, run: string, uploads: Uploads, ctx: Ctx): Promise { + for (const entry of await readdir(uploadsDir).catch(() => [])) { + if (join(uploadsDir, entry) !== run) await rm(join(uploadsDir, entry), { recursive: true, force: true }); + } + try { + for (;;) { + await sleep(SWEEP_MS, undefined, { signal: ctx.abortSignal }); + await uploads.expire(UPLOAD_IDLE_MS); + } + } finally { + if (ctx.abortSignal?.aborted === true) await uploads.close(); + } +} + +/** A cap refused a write. */ +function tooLarge(message: string): ChordError { + return new ChordError({ code: "too_large", message: `files: ${message}` }); +} diff --git a/packages/chord/src/next-node/app/index.ts b/packages/chord/src/next-node/app/index.ts new file mode 100644 index 000000000..2eb6e1b4b --- /dev/null +++ b/packages/chord/src/next-node/app/index.ts @@ -0,0 +1,24 @@ +// Chord v2's app kit (`@earendil-works/chord/next/app` once v2 has its entry point; slice 13): `startServer` on the server +// and `startWorker` in a worker's entry, the start-data objects, sessions and the owner's sign-in, device pairing, the +// files of each host, export and import per address, and the optional plugins (users, invites, ephemeral login). + +export { + type App, + type AppApi, + type AppError, + type AppKind, + type AppOptions, + type AppPlugins, + type AppWorkerKind, + Server, + startServer, +} from "./app.ts"; +export { ARCHIVE_FORMAT, type ArchiveManifest } from "./archive.ts"; +export { type EphemeralUser, ephemeralLoginFacet } from "./ephemeral.ts"; +export { type FileInfo, Files, type FilesCaps } from "./files.ts"; +export { AppInfo, WorkerInfo } from "./info.ts"; +export { inviteFacet } from "./invite.ts"; +export { OWNER, type OwnerPrincipal } from "./owner.ts"; +export { SESSION_COOKIE, type SessionInfo, type SessionKind, Sessions } from "./sessions.ts"; +export { defineUsers } from "./users.ts"; +export { startWorker, type WorkerOptions } from "./worker.ts"; diff --git a/packages/chord/src/next-node/app/info.ts b/packages/chord/src/next-node/app/info.ts new file mode 100644 index 000000000..c1c0ba148 --- /dev/null +++ b/packages/chord/src/next-node/app/info.ts @@ -0,0 +1,52 @@ +// The start-data objects (slice 13 §1; decision "The plugin model, points 3, 4, 5 and 8" (8)): base code shrinks to +// the API, the bootstrap and one facet serving process values as core objects plugins `get`. `AppInfo` on the +// server (the data directory, the app's name and its API version), `WorkerInfo` on a worker (its address and the +// app's start data for it). + +import type { JsonValue } from "../../json.ts"; +import { local, obj, str, unknown } from "../../next/contracts/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import { constant } from "../../next/state/index.ts"; + +/** What the server knows of the app: its data directory, its name, its newest API version. */ +export const AppInfo = local("chord.app.info", { + version: 1, + params: {}, + members: { dataDir: str(), name: str(), version: str() }, +}); + +/** What a worker knows of itself: its address (kind, params, key) and the app's start data for it. */ +export const WorkerInfo = local("chord.app.worker-info", { + version: 1, + params: {}, + members: { + address: obj({ kind: str(), key: str(), params: unknown() }), + start: unknown(), + }, +}); + +/** The facet serving `AppInfo` with `info`. */ +export function appInfoFacet

(info: { + readonly dataDir: string; + readonly name: string; + readonly version: string; +}): Facet

{ + return defineFacet

((host) => + host.serve(AppInfo, () => ({ + dataDir: constant(info.dataDir), + name: constant(info.name), + version: constant(info.version), + })), + ); +} + +/** The facet serving `WorkerInfo` with the worker's address and start data. */ +export function workerInfoFacet

(info: { + readonly address: { readonly kind: string; readonly key: string; readonly params: JsonValue }; + readonly start: JsonValue; +}): Facet

{ + return defineFacet

((host) => + host.serve(WorkerInfo, () => ({ address: constant(info.address), start: constant(info.start) })), + ); +} diff --git a/packages/chord/src/next-node/app/invite.ts b/packages/chord/src/next-node/app/invite.ts new file mode 100644 index 000000000..5955c7cb1 --- /dev/null +++ b/packages/chord/src/next-node/app/invite.ts @@ -0,0 +1,72 @@ +// The invite plugin (slice 13 §3, decision "The app kit, point 2"): the owner makes an invite link for a principal of +// the app it chooses (`POST /_chord/invites`, owner only: `{ principal, label }` → `{ path }`); the invitee opens it, +// its page posts the token (the URL's fragment, never logged) to `POST /_chord/invite`, which consumes it (single use, +// 7 days) and answers with a browser session for that principal (§2's sessions). Invites wait in memory: a restart +// drops the ones not used. An exported facet, as `defineUsers` (see there why). + +import type { JsonValue } from "../../json.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import { Http } from "../http/index.ts"; +import { fragmentPage } from "./owner.ts"; +import { hashToken, newToken } from "./secrets.ts"; +import { Sessions } from "./sessions.ts"; + +/** How long an invite is valid: 7 days. */ +const INVITE_MS = 7 * 24 * 60 * 60_000; + +/** The invite link's path: its page, and the post that consumes it. */ +const INVITE_PATH = "/_chord/invite"; + +/** An invite waiting: the principal it signs in, its label, its expiry. */ +type Invite = { readonly principal: JsonValue; readonly label: string; readonly expires: number }; + +/** The invite plugin's server facet. */ +export function inviteFacet

(): Facet

{ + // By their token's hash: what the map holds is no credential. + const invites = new Map(); + return defineFacet

((_host, self) => + self.contribute(Http, { + routes: { + "POST /_chord/invites": async (request, ctx) => { + if (ctx.principal?.kind !== "owner") return { kind: "error", status: 403 }; + // A trust boundary: the owner's request, checked for its shape. + const body = (await request.json()) as { + readonly principal?: JsonValue; + readonly label?: unknown; + } | null; + const principal = body?.principal; + if (typeof principal !== "object" || principal === null || typeof body?.label !== "string") { + return { kind: "error", status: 400 }; + } + const token = newToken(); + invites.set(hashToken(token), { principal, label: body.label, expires: Date.now() + INVITE_MS }); + return { kind: "json", value: { path: `${INVITE_PATH}#${token}` } }; + }, + [`GET ${INVITE_PATH}`]: { + public: true, + handler: async (request) => ({ kind: "html", body: fragmentPage(INVITE_PATH, request.nonce) }), + }, + [`POST ${INVITE_PATH}`]: { + public: true, + handler: async (request, ctx) => { + // A trust boundary: anyone may post here. + const body = (await request.json()) as { readonly token?: unknown } | null; + const hash = typeof body?.token === "string" ? hashToken(body.token) : undefined; + const invite = hash === undefined ? undefined : invites.get(hash); + if (hash === undefined || invite === undefined) return { kind: "error", status: 403 }; + invites.delete(hash); + if (invite.expires <= Date.now()) return { kind: "error", status: 403 }; + const sessions = await ctx.get(Sessions, {}); + const secure = request.provenance.kind !== "local"; + const issued = await sessions.issue( + { principal: invite.principal, label: invite.label, kind: "browser", secure }, + ctx, + ); + return { kind: "json", value: "signed in", headers: { "Set-Cookie": issued.cookie } }; + }, + }, + }, + }), + ); +} diff --git a/packages/chord/src/next-node/app/layout.ts b/packages/chord/src/next-node/app/layout.ts new file mode 100644 index 000000000..ad3b379fa --- /dev/null +++ b/packages/chord/src/next-node/app/layout.ts @@ -0,0 +1,105 @@ +// The app kit's data directory (slice 13 §4, decision "The app kit, point 4: storage"): one fixed layout, made private +// at every start. `server/` holds the server's store and files, `workers//

/` each worker's store and +// files, `plugins/` the user's plugins root, `logs/` the logs, `owner-token` the owner's token for command-line +// clients. Directories are 0700 and stores 0600, also when they existed looser before (other local users must not +// read the sessions in a store, nor replace it). + +import { createHash } from "node:crypto"; +import { chmod, mkdir, open, realpath, stat } from "node:fs/promises"; +import { homedir } from "node:os"; +import { join, parse } from "node:path"; +import type { Address } from "../../next/contracts/index.ts"; + +/** The paths of a data directory. */ +export type AppLayout = { + readonly dataDir: string; + readonly server: string; + readonly serverStore: string; + readonly workers: string; + readonly plugins: string; + readonly logs: string; + readonly ownerToken: string; + readonly cookieSecret: string; +}; + +/** The data directory of app `name` unless the start script names one: `~/.`. */ +export function defaultDataDir(name: string): string { + return join(homedir(), `.${name}`); +} + +/** + * The real path of the data directory `path`, made when missing (mode 0700), refused when the kit must not make it + * private: the home directory, a file system's root, or a directory another user owns (a clear error, instead of + * tightening a directory others use or failing on `EPERM`). Its mode is set on the real path, never through a link. + */ +export async function openDataDir(path: string): Promise { + await mkdir(path, { recursive: true, mode: 0o700 }); + const dataDir = await realpath(path); + const home = await realpath(homedir()); + if (dataDir === home || dataDir === parse(dataDir).root) { + const what = dataDir === home ? "the home directory" : "a file system's root"; + throw new Error(`startServer: the data directory ${dataDir} is ${what}; name a directory of the app's own`); + } + const { uid } = await stat(dataDir); + if (process.getuid !== undefined && uid !== process.getuid()) { + throw new Error(`startServer: the data directory ${dataDir} belongs to another user (uid ${uid})`); + } + return dataDir; +} + +/** The layout of `dataDir` (a real path, `openDataDir`'s), its directories made private (`privateDirectory`). */ +export async function createLayout(dataDir: string): Promise { + await privateDirectory(dataDir); + const layout: AppLayout = { + dataDir, + server: join(dataDir, "server"), + serverStore: join(dataDir, "server", "store.sqlite"), + workers: join(dataDir, "workers"), + plugins: join(dataDir, "plugins"), + logs: join(dataDir, "logs"), + ownerToken: join(dataDir, "owner-token"), + cookieSecret: join(dataDir, "server", "cookie-secret"), + }; + for (const dir of [layout.server, layout.workers, layout.plugins, layout.logs]) await privateDirectory(dir); + return layout; +} + +/** Makes `dir` (and missing parents) with mode 0700, and sets 0700 on it when it existed. */ +export async function privateDirectory(dir: string): Promise { + await mkdir(dir, { recursive: true, mode: 0o700 }); + await chmod(dir, 0o700); +} + +/** + * Makes the store file `path` private before SQLite opens it: created empty with mode 0600 when missing (SQLite gives + * its journal files the database's mode), else set to 0600 with the journal files it has. + */ +export async function privateStore(path: string): Promise { + await (await open(path, "a", 0o600)).close(); + for (const file of [path, `${path}-wal`, `${path}-shm`]) { + await chmod(file, 0o600).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }); + } +} + +/** The directory of the worker at `address`: `workers//`. */ +export function workerDirectory(dataDir: string, address: Address): string { + return join(dataDir, "workers", address.kind, encodedKey(address.key)); +} + +/** The longest key kept readable as a directory name; a longer one is named by its hash. */ +const READABLE_KEY = 160; + +/** + * An address key as one directory name: percent-encoded (`/`, `\`, `.` and every other character a file system might + * treat specially included), or its SHA-256 when that is longer than `READABLE_KEY`. Capital letters are encoded too: + * on a case-insensitive file system (macOS) `p|A` and `p|a` would otherwise share a directory. + */ +function encodedKey(key: string): string { + const encoded = encodeURIComponent(key).replace( + /(%[0-9A-F]{2})|[.!'()*~A-Z]/g, + (match, escaped: string | undefined) => escaped ?? `%${match.charCodeAt(0).toString(16).toUpperCase()}`, + ); + return encoded.length <= READABLE_KEY ? encoded : `sha256-${createHash("sha256").update(key).digest("hex")}`; +} diff --git a/packages/chord/src/next-node/app/owner.ts b/packages/chord/src/next-node/app/owner.ts new file mode 100644 index 000000000..2c0179f43 --- /dev/null +++ b/packages/chord/src/next-node/app/owner.ts @@ -0,0 +1,123 @@ +// The owner (slice 13 §2, decision "The app kit, point 3: authentication"): no owner by loopback alone. At each start +// the kit makes a one-time owner link (a random token, single use, valid 10 minutes, the previous one dropped); the +// start script prints it. Its page reads the token from the URL's fragment (never sent to a server, so never in a +// log) and posts it; the post consumes it and sets a browser session cookie for the owner. The owner token file +// (mode 0600) carries a device session of the owner for command-line clients on the same machine. + +import { chmod } from "node:fs/promises"; +import type { Context } from "../../context/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet, type Host } from "../../next/host/index.ts"; +import { Http } from "../http/index.ts"; +import { hashToken, matchesHash, newToken, readSecret, writeSecret } from "./secrets.ts"; +import { Sessions } from "./sessions.ts"; + +/** The kit's owner principal: a variant every app's principal type has. */ +export type OwnerPrincipal = { readonly kind: "owner" }; + +/** The owner principal. */ +export const OWNER: OwnerPrincipal = { kind: "owner" }; + +/** How long an owner link is valid: 10 minutes. */ +const OWNER_LINK_MS = 10 * 60_000; + +/** The owner link's one-time token: made at each start (the previous one dropped), consumed by its first use. */ +export class OwnerLink { + #hash: string | undefined; + #expires = 0; + + /** A fresh token (the previous one no longer valid), valid for 10 minutes. */ + renew(): string { + const token = newToken(); + this.#hash = hashToken(token); + this.#expires = Date.now() + OWNER_LINK_MS; + return token; + } + + /** Whether `token` is the current one and has not expired; it is used up either way when it matches. */ + consume(token: string): boolean { + const hash = this.#hash; + if (hash === undefined || !matchesHash(token, hash)) return false; + this.#hash = undefined; + return Date.now() < this.#expires; + } +} + +/** The path of the owner link's page and post. */ +export const OWNER_PATH = "/_chord/owner"; + +/** + * The kit's facet for the owner link: `GET /_chord/owner`, a page posting the fragment's token, and `POST + * /_chord/owner`, which consumes it and answers with the owner's session cookie. + */ +export function ownerFacet

(link: OwnerLink): Facet

{ + return defineFacet

((_host, self) => + self.contribute(Http, { + routes: { + [`GET ${OWNER_PATH}`]: { + public: true, + handler: async (request) => ({ kind: "html", body: fragmentPage(OWNER_PATH, request.nonce) }), + }, + [`POST ${OWNER_PATH}`]: { + public: true, + handler: async (request, ctx) => { + const body = (await request.json()) as { readonly token?: unknown } | null; + // A trust boundary: anyone may post here. + if (typeof body?.token !== "string" || !link.consume(body.token)) + return { kind: "error", status: 403 }; + const sessions = await ctx.get(Sessions, {}); + const secure = request.provenance.kind !== "local"; + const issued = await sessions.issue( + { principal: OWNER, label: "owner browser", kind: "browser", secure }, + ctx, + ); + return { kind: "json", value: "signed in", headers: { "Set-Cookie": issued.cookie } }; + }, + }, + }, + }), + ); +} + +/** + * A sign-in page for a link with a token in its fragment (the owner link, an invite): its script (the response's nonce) + * posts the token to `path`, then opens the app. + */ +export function fragmentPage(path: string, nonce: string): string { + return [ + "Sign in

Signing in…

", + `", + ].join("\n"); +} + +/** + * The owner token file at `path`: kept when its token still authenticates as the owner (its session exists, its secret + * matches, it has not expired; its mode set to 0600 again), else a new device session of the owner written there (mode + * 0600, in one step). Its token, for the start script. + */ +export async function ownerTokenFile

( + path: string, + host: Host

, + context: Context, +): Promise { + const existing = await readSecret(path); + return host.root(async (ctx) => { + const sessions = await ctx.get(Sessions, {}); + if (existing !== undefined && isOwner(await sessions.verify(existing, ctx))) { + await chmod(path, 0o600); + return existing; + } + const { token } = await sessions.issue({ principal: OWNER, label: "owner token file", kind: "device" }, ctx); + await writeSecret(path, token, existing === undefined ? "create" : "replace"); + return token; + }, context); +} + +/** Whether `principal` (as `Sessions.verify` gives it) is the owner. */ +function isOwner(principal: unknown): boolean { + return typeof principal === "object" && principal !== null && (principal as { kind?: unknown }).kind === "owner"; +} diff --git a/packages/chord/src/next-node/app/pairing.ts b/packages/chord/src/next-node/app/pairing.ts new file mode 100644 index 000000000..8741aa5cd --- /dev/null +++ b/packages/chord/src/next-node/app/pairing.ts @@ -0,0 +1,205 @@ +// Device pairing (slice 13 §2, decision "The app kit, point 3" (c)): a command-line or terminal client on another +// machine asks for a code (`POST /_chord/pair`, public) and polls with a secret it got with it; the owner opens +// `/_chord/pair/` in a signed-in browser and approves; the next poll returns a device session's token, once. +// Codes are short, random, single use and expire after 10 minutes; the poll secret is kept only hashed. Asking is +// public, so pairings anonymous clients start are capped: `maxPending` in all (100 by default) and `perAddress` per +// client address (10). The trade: a party filling the caps locks pairing out for others for up to 10 minutes (behind a +// tunnel every remote client has the tunnel's address), instead of growing the server's memory without bound; a +// signed-in owner's pairings are not counted, so the owner can always pair. + +import { randomInt } from "node:crypto"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import { Http, type HttpRequest, type HttpResponse } from "../http/index.ts"; +import { OWNER } from "./owner.ts"; +import { hashToken, newToken } from "./secrets.ts"; +import { Sessions } from "./sessions.ts"; + +/** How long a pairing code is valid: 10 minutes. */ +const PAIRING_MS = 10 * 60_000; + +/** The code's letters: no 0/O, 1/I/L. */ +const ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789"; + +/** Who starts a pairing: the signed-in owner (not counted), or an anonymous client at an address (counted). */ +export type PairingStarter = { readonly kind: "owner" } | { readonly kind: "anonymous"; readonly address: string }; + +/** One pairing under way: its code, the device's label, who started it, its expiry, its token once approved. */ +type Pairing = { + readonly code: string; + readonly label: string; + readonly by: PairingStarter; + readonly expires: number; + token?: string; +}; + +/** The limits of pairings under way: in all (100 by default) and per client address (10 by default). */ +export type PairingLimits = { readonly maxPending?: number; readonly perAddress?: number }; + +/** The pairings under way, by code and by their poll secret's hash, with the anonymous ones counted. */ +export class Pairings { + readonly #byCode = new Map(); + readonly #byPoll = new Map(); + readonly #perAddress = new Map(); + #anonymous = 0; + readonly #maxPending: number; + readonly #maxPerAddress: number; + + constructor(limits: PairingLimits = {}) { + this.#maxPending = limits.maxPending ?? 100; + this.#maxPerAddress = limits.perAddress ?? 10; + } + + /** + * A new pairing for a device labelled `label`, started `by` the owner or an anonymous client: its code (for the + * owner) and its poll secret (for the device); `undefined` when an anonymous client reached a limit. + */ + begin(label: string, by: PairingStarter): { readonly code: string; readonly poll: string } | undefined { + this.#expire(); + if (by.kind === "anonymous") { + const full = this.#anonymous >= this.#maxPending; + if (full || (this.#perAddress.get(by.address) ?? 0) >= this.#maxPerAddress) return undefined; + this.#anonymous++; + this.#perAddress.set(by.address, (this.#perAddress.get(by.address) ?? 0) + 1); + } + let code: string; + do code = `${letters(4)}-${letters(4)}`; + while (this.#byCode.has(code)); + const poll = newToken(); + const pairing: Pairing = { code, label, by, expires: Date.now() + PAIRING_MS }; + this.#byCode.set(code, pairing); + this.#byPoll.set(hashToken(poll), pairing); + return { code, poll }; + } + + /** The pairing waiting for approval under `code` (any case), unexpired. */ + waiting(code: string): Pairing | undefined { + this.#expire(); + const pairing = this.#byCode.get(code.toUpperCase()); + return pairing?.token === undefined ? pairing : undefined; + } + + /** What a poll with `poll` learns: still waiting, the device's token (once: the pairing ends), or nothing. */ + poll( + poll: string, + ): { readonly kind: "waiting" } | { readonly kind: "approved"; readonly token: string } | undefined { + this.#expire(); + const hash = hashToken(poll); + const pairing = this.#byPoll.get(hash); + if (pairing === undefined) return undefined; + if (pairing.token === undefined) return { kind: "waiting" }; + this.#end(hash, pairing); + return { kind: "approved", token: pairing.token }; + } + + /** Drops the expired pairings. */ + #expire(): void { + const now = Date.now(); + for (const [hash, pairing] of this.#byPoll) { + if (pairing.expires <= now) this.#end(hash, pairing); + } + } + + /** Ends `pairing` (its poll secret's hash `hash`). */ + #end(hash: string, pairing: Pairing): void { + this.#byPoll.delete(hash); + this.#byCode.delete(pairing.code); + const { by } = pairing; + if (by.kind === "owner") return; + this.#anonymous--; + const left = (this.#perAddress.get(by.address) ?? 1) - 1; + if (left === 0) this.#perAddress.delete(by.address); + else this.#perAddress.set(by.address, left); + } +} + +/** `count` random letters of `ALPHABET` (`crypto.randomInt`). */ +function letters(count: number): string { + let text = ""; + for (let index = 0; index < count; index++) text += ALPHABET[randomInt(ALPHABET.length)]; + return text; +} + +/** The pairing routes' prefix. */ +export const PAIR_PATH = "/_chord/pair"; + +/** + * The kit's facet for device pairing: `POST /_chord/pair` (public: `{ label? }` → `{ code, poll }`, 429 past a limit + * unless the owner asks), `POST /_chord/pair/poll` (public: `{ poll }` → `{ status: "waiting" }`, `{ status: + * "approved", token }` once, else 404), `GET` and `POST /_chord/pair/:code` (the owner's confirm page and approval: a + * device session of the owner). + */ +export function pairingFacet

(pairings: Pairings): Facet

{ + return defineFacet

((_host, self) => + self.contribute(Http, { + routes: { + [`POST ${PAIR_PATH}`]: { + public: true, + handler: async (request, ctx): Promise => { + const label = await labelOf(request); + const by: PairingStarter = + ctx.principal?.kind === "owner" + ? { kind: "owner" } + : { kind: "anonymous", address: request.remoteAddress ?? "" }; + const begun = pairings.begin(label, by); + return begun === undefined ? { kind: "error", status: 429 } : { kind: "json", value: begun }; + }, + }, + [`POST ${PAIR_PATH}/poll`]: { + public: true, + handler: async (request): Promise => { + // A trust boundary: anyone may poll. + const body = (await request.json()) as { readonly poll?: unknown } | null; + const polled = typeof body?.poll === "string" ? pairings.poll(body.poll) : undefined; + if (polled === undefined) return { kind: "error", status: 404 }; + if (polled.kind === "waiting") return { kind: "json", value: { status: "waiting" } }; + return { kind: "json", value: { status: "approved", token: polled.token } }; + }, + }, + [`GET ${PAIR_PATH}/:code`]: async (request, ctx) => { + if (ctx.principal?.kind !== "owner") return { kind: "error", status: 403 }; + const pairing = pairings.waiting(request.params.code ?? ""); + if (pairing === undefined) + return { kind: "html", status: 404, body: "

No such pairing (or it expired).

" }; + return { kind: "html", body: confirmPage(pairing.code, pairing.label) }; + }, + [`POST ${PAIR_PATH}/:code`]: async (request, ctx) => { + if (ctx.principal?.kind !== "owner") return { kind: "error", status: 403 }; + const pairing = pairings.waiting(request.params.code ?? ""); + if (pairing === undefined) + return { kind: "html", status: 404, body: "

No such pairing (or it expired).

" }; + const sessions = await ctx.get(Sessions, {}); + const { token } = await sessions.issue({ principal: OWNER, label: pairing.label, kind: "device" }, ctx); + pairing.token = token; + return { + kind: "html", + body: `

Approved: ${escapeHtml(pairing.label)} is signed in as the owner.

`, + }; + }, + }, + }), + ); +} + +/** The device's label from a pairing request's body (`{ label }`, at most 100 characters), else `device`. */ +async function labelOf(request: HttpRequest): Promise { + const declared = Number(request.headers.get("content-length") ?? "0"); + if (declared === 0 && request.headers.get("transfer-encoding") === null) return "device"; + // A trust boundary: anyone may ask for a code. + const body = (await request.json(4096)) as { readonly label?: unknown } | null; + return typeof body?.label === "string" && body.label.trim() !== "" ? body.label.trim().slice(0, 100) : "device"; +} + +/** The owner's confirm page for pairing `code` (a form posting to itself: same origin, the session cookie). */ +function confirmPage(code: string, label: string): string { + return [ + "Pair a device", + `

Sign in ${escapeHtml(label)} as the owner? Its code: ${escapeHtml(code)} (compare it with the device).

`, + '
', + ].join("\n"); +} + +/** `text` safe in HTML. */ +function escapeHtml(text: string): string { + return text.replace(/[&<>"']/g, (char) => `&#${char.charCodeAt(0)};`); +} diff --git a/packages/chord/src/next-node/app/secrets.ts b/packages/chord/src/next-node/app/secrets.ts new file mode 100644 index 000000000..6de1d870f --- /dev/null +++ b/packages/chord/src/next-node/app/secrets.ts @@ -0,0 +1,75 @@ +// Secrets of the app kit (slice 13 §2): tokens from `crypto.randomBytes`, kept only as their SHA-256 hashes and +// compared in constant time, and secret files (the cookie key, the owner token file) created with mode 0600 in one +// step, so no other OS user and no half-written file ever reads them. + +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { link, open, readFile, rename, unlink } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +/** A fresh random token of `bytes` bytes, base64url. */ +export function newToken(bytes = 32): string { + return randomBytes(bytes).toString("base64url"); +} + +/** The hash a token is kept as (SHA-256, hex): what leaks from a store is no credential. */ +export function hashToken(token: string): string { + return createHash("sha256").update(token).digest("hex"); +} + +/** Whether `token` hashes to `hash`, compared in constant time. */ +export function matchesHash(token: string, hash: string): boolean { + const given = Buffer.from(hashToken(token), "hex"); + const kept = Buffer.from(hash, "hex"); + return given.length === kept.length && timingSafeEqual(given, kept); +} + +/** The secret in the file at `path`, created (mode 0600, in one step) with a fresh token when it does not exist yet. */ +export async function secretFile(path: string): Promise { + const existing = await readSecret(path); + if (existing !== undefined) return existing; + const secret = newToken(); + const written = await writeSecret(path, secret, "create"); + return written === "exists" ? ((await readSecret(path)) ?? secret) : secret; +} + +/** The text of the secret file at `path`, or `undefined` when it does not exist. */ +export async function readSecret(path: string): Promise { + try { + return (await readFile(path, "utf8")).trim(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw error; + } +} + +/** + * Writes `text` to `path` with mode 0600 in one step: a temporary file of the same directory (made exclusively, 0600) + * linked into place (`create`: refused when `path` exists, `"exists"` returned) or renamed over it (`replace`). + */ +export async function writeSecret( + path: string, + text: string, + mode: "create" | "replace", +): Promise<"written" | "exists"> { + const temporary = join(dirname(path), `.${newToken(9)}.tmp`); + const handle = await open(temporary, "wx", 0o600); + try { + await handle.writeFile(`${text}\n`); + await handle.sync(); + } finally { + await handle.close(); + } + try { + if (mode === "replace") { + await rename(temporary, path); + return "written"; + } + await link(temporary, path); + return "written"; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") return "exists"; + throw error; + } finally { + await unlink(temporary).catch(() => {}); + } +} diff --git a/packages/chord/src/next-node/app/sessions.ts b/packages/chord/src/next-node/app/sessions.ts new file mode 100644 index 000000000..88ce03bba --- /dev/null +++ b/packages/chord/src/next-node/app/sessions.ts @@ -0,0 +1,278 @@ +// Sessions (slice 13 §2, decision "The app kit, point 3: authentication"): the kit's object on the server that issues +// and revokes sessions, and its authenticator. A session is a principal behind a token `.` (random); only +// the secret's hash is kept. A browser carries its token in the kit's signed session cookie (`HttpOnly`, +// `SameSite=Strict`), a command-line client in `Authorization: Bearer `. Browser and device sessions persist in +// the server's store and expire after 400 days unused; ephemeral sessions (a pick-a-name login, a public route) live +// in memory only (`EphemeralSessions`), at most `maxEphemeral` of them (past it the oldest ends), so a public login +// costs the same whatever came before; each ends with its client's connection (its last link closed, a grace period +// passed: the links endpoint's `opened` hook counts them), and none outlives the server's run. + +import type { Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { arr, bool, local, map, method, num, obj, oneOf, opt, str, unknown } from "../../next/contracts/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineState } from "../../next/documents/index.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import { + Authenticators, + cookie, + Http, + type HttpRequest, + type HttpWrap, + type Provenance, + readCookie, + readSignedCookie, +} from "../http/index.ts"; +import { EphemeralSessions } from "./ephemeral-sessions.ts"; +import { hashToken, matchesHash, newToken } from "./secrets.ts"; + +/** What a session is for: a browser (its cookie), a device (a command-line client's token), an ephemeral login. */ +export type SessionKind = "browser" | "device" | "ephemeral"; + +/** A browser or device session as `Sessions.list` shows it: never its token or hash. */ +export type SessionInfo = { + readonly id: string; + readonly label: string; + readonly kind: "browser" | "device"; + readonly principal: unknown; + readonly created: number; + readonly lastUsed: number; +}; + +/** The kit's sessions (one per server): issue, verify, revoke, and the watchable list of persisted ones. */ +export const Sessions = local("chord.app.sessions", { + version: 1, + params: {}, + members: { + /** The browser and device sessions (ephemeral ones are not listed: there may be many, and they are short). */ + list: arr( + obj({ + id: str(), + label: str(), + kind: oneOf("browser", "device"), + principal: unknown(), + created: num(), + lastUsed: num(), + }), + ), + /** + * A new session for `principal`: its id, its token (shown once, kept only hashed) and the session cookie carrying + * it for a browser (`Set-Cookie`; `secure` for a remote one). + */ + issue: method( + obj({ + principal: unknown(), + label: str(), + kind: oneOf("browser", "device", "ephemeral"), + secure: opt(bool()), + }), + obj({ id: str(), token: str(), cookie: str() }), + ), + /** The principal `token` authenticates as, or `null` (unknown, revoked, expired, malformed). */ + verify: method(str(), unknown()), + /** Ends session `id` at once: its token authenticates nothing more. */ + revoke: method(str()), + }, +}); + +/** The name of the kit's session cookie. */ +export const SESSION_COOKIE = "chord-session"; + +/** The stored sessions: by id, each with its secret's hash and its principal (as JSON text: any of the app's). */ +const SessionsDocument = defineState("chord.app.sessions", { + schema: obj({ + sessions: map( + obj({ + hash: str(), + label: str(), + kind: oneOf("browser", "device"), + principal: str(), + created: num(), + lastUsed: num(), + }), + ), + }), + version: 1, + initial: () => ({ sessions: {} }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 50, +}); + +/** How often a session's `lastUsed` is written at most: once a minute (not one commit per request). */ +const TOUCH_MS = 60_000; + +/** How long a persisted session lives unused: 400 days, as its cookie. */ +const SESSION_IDLE_MS = 400 * 24 * 60 * 60_000; + +/** The sessions' limits: how many ephemeral sessions live at once (1,000 by default; past it the oldest ends). */ +export type SessionLimits = { readonly maxEphemeral?: number }; + +/** The kit's sessions: the facet for the server's core facets, and the links endpoint's `opened` hook. */ +export type KitSessions

= { + readonly facet: Facet

; + /** A link opened with `request`'s credentials: an ephemeral session's links are counted (it ends with its last). */ + readonly opened: (request: HttpRequest) => (() => void) | undefined; +}; + +/** + * The kit's sessions: its facet serving `Sessions` (a host-lifetime object) and contributing its authenticator (the + * bearer token first, else the session cookie signed with `cookieSecret`), and the links hook ending an ephemeral + * session with its client's connection. + */ +export function createSessions

(cookieSecret: string, limits: SessionLimits = {}): KitSessions

{ + const ephemeral = new EphemeralSessions(limits.maxEphemeral ?? 1000); + const opened = (request: HttpRequest): (() => void) | undefined => { + const token = bearerToken(request) ?? readCookie(request, SESSION_COOKIE, cookieSecret); + const dot = token?.indexOf(".") ?? -1; + return dot > 0 ? ephemeral.linkOpened(token!.slice(0, dot)) : undefined; + }; + return { facet: sessionsFacet

(cookieSecret, ephemeral), opened }; +} + +/** The facet of `createSessions`. */ +function sessionsFacet

(cookieSecret: string, ephemeral: EphemeralSessions): Facet

{ + return defineFacet

((host) => + host.serve(Sessions, (_params, self) => { + const document = self.state(SessionsDocument); + const change = (fn: (sessions: Record) => void, context: Context) => + // Not awaited by authentication: the request goes on; a write that fails is made again by a later request. + void document.change((draft) => fn(draft.sessions), context).catch(() => {}); + /** The principal of `token`, or `undefined`; touches or lazily removes the persisted session it names. */ + const authenticate = (token: string, context: Context): Principal | undefined => { + const dot = token.indexOf("."); + if (dot <= 0) return undefined; + const id = token.slice(0, dot); + const secret = token.slice(dot + 1); + const short = ephemeral.get(id); + if (short !== undefined) return matchesHash(secret, short.hash) ? short.principal : undefined; + // A trust boundary: `id` is the client's (`Object.hasOwn`: no inherited member is a session). + const sessions = document.value.sessions; + if (!Object.hasOwn(sessions, id)) return undefined; + const session = sessions[id]!; + if (!matchesHash(secret, session.hash)) return undefined; + const now = Date.now(); + if (now - session.lastUsed > SESSION_IDLE_MS) { + change((draft) => { + delete draft[id]; + }, context); + return undefined; + } + if (now - session.lastUsed > TOUCH_MS) { + change((draft) => { + if (Object.hasOwn(draft, id)) draft[id]!.lastUsed = now; + }, context); + } + // Stored by `issue` from a principal of the app's type. + return JSON.parse(session.principal) as Principal; + }; + self.contribute(Authenticators, (request, context) => { + const token = bearerToken(request) ?? readCookie(request, SESSION_COOKIE, cookieSecret); + return token === undefined ? undefined : authenticate(token, context); + }); + self.contribute(Http, { wraps: [rollingCookie(cookieSecret, authenticate)] }); + return { + list: self.derive([document], ([value]) => + Object.entries(value.sessions).map(([id, { label, kind, principal, created, lastUsed }]) => ({ + id, + label, + kind, + // Stored by `issue` as JSON text. + principal: JSON.parse(principal) as JsonValue, + created, + lastUsed, + })), + ), + issue: async ({ principal, label, kind, secure }, ctx) => { + const id = newToken(12); + const secret = newToken(32); + if (kind === "ephemeral") { + // Principals are the app's: `issue`'s callers pass one of its type. + ephemeral.issue(id, hashToken(secret), principal as Principal); + } else { + const now = Date.now(); + const stored = JSON.stringify(principal); + await document.change((draft) => { + draft.sessions[id] = { + hash: hashToken(secret), + label, + kind, + principal: stored, + created: now, + lastUsed: now, + }; + }, ctx); + } + const token = `${id}.${secret}`; + // Only whether the browser is remote matters to the cookie (`Secure`). + const provenance: Provenance = secure === true ? { kind: "remote", host: "" } : { kind: "local" }; + const set = sessionCookie(token, kind, { provenance }, cookieSecret); + return { id, token, cookie: set }; + }, + verify: async (token, ctx) => (authenticate(token, ctx) as JsonValue | undefined) ?? null, + revoke: async (id, ctx) => { + if (ephemeral.revoke(id)) return null; + await document.change((draft) => { + delete draft.sessions[id]; + }, ctx); + return null; + }, + }; + }), + ); +} + +/** + * The session cookie carrying `token` (a session of `kind`) for a browser of `request`'s provenance: a persisted + * session's lives as long as the session does unused (400 days, renewed by `rollingCookie`), an ephemeral one's only + * while the browser runs. + */ +function sessionCookie( + token: string, + kind: SessionKind, + request: Pick, + secret: string, +): string { + const maxAge = kind === "ephemeral" ? undefined : SESSION_IDLE_MS / 1000; + return cookie(SESSION_COOKIE, token, { secret, request, maxAge }); +} + +/** How old a session cookie gets before a response renews it: a day. */ +const ROLL_MS = 24 * 60 * 60_000; + +/** + * The wrap around every route renewing the session cookie a request signed in with once it is older than a day: the + * response sets it again with a fresh signed expiry, so a browser used within every 400 days stays signed in. Not when + * the route sets a cookie itself, nor for an ephemeral session's (it has no expiry) or a token that no longer + * authenticates. + */ +function rollingCookie( + secret: string, + authenticate: (token: string, context: Context) => Principal | undefined, +): HttpWrap { + return { + name: "chord.app.rolling-cookie", + phase: "headers", + scope: "all", + wrap: async (request, next, ctx) => { + const response = await next(request); + if (ctx.principal === undefined || Object.keys(response.headers ?? {}).some(isSetCookie)) return response; + const signed = readSignedCookie(request, SESSION_COOKIE, secret); + if (signed?.expires === undefined || signed.expires - Date.now() > SESSION_IDLE_MS - ROLL_MS) return response; + if (authenticate(signed.value, ctx) === undefined) return response; + const renewed = sessionCookie(signed.value, "browser", request, secret); + return { ...response, headers: { ...response.headers, "Set-Cookie": renewed } }; + }, + }; +} + +/** Whether a header name is `Set-Cookie` (any case). */ +function isSetCookie(name: string): boolean { + return name.toLowerCase() === "set-cookie"; +} + +/** The token of an `Authorization: Bearer ` header. */ +function bearerToken(request: HttpRequest): string | undefined { + const header = request.headers.get("authorization"); + const match = header === null ? null : /^Bearer\s+(\S+)$/i.exec(header); + return match?.[1]; +} diff --git a/packages/chord/src/next-node/app/tar.ts b/packages/chord/src/next-node/app/tar.ts new file mode 100644 index 000000000..7c1cde0ad --- /dev/null +++ b/packages/chord/src/next-node/app/tar.ts @@ -0,0 +1,181 @@ +// A minimal ustar (POSIX tar) writer and reader for the kit's address archives (slice 13 §4): regular files and +// directories, paths of at most 255 bytes, nothing else. The reader is a trust boundary: an archive may come from +// anywhere, so it refuses every entry that is no regular file or directory, a path that is absolute, holds `..`, a +// backslash or NUL, or names an empty segment, and a header whose checksum does not hold. + +/** One entry to write: its path (`/`-separated, relative) and its bytes, as a size and the chunks that make it up. */ +export type TarEntry = { readonly path: string; readonly size: number; readonly data: AsyncIterable }; + +/** One entry read: a regular file (its data, to be consumed before the next entry) or a directory. */ +export type TarRead = + | { readonly kind: "file"; readonly path: string; readonly size: number; readonly data: AsyncIterable } + | { readonly kind: "directory"; readonly path: string }; + +/** A tar block. */ +const BLOCK = 512; + +/** The archive of `entries`, as the chunks of its bytes (two zero blocks end it). */ +export async function* writeTar(entries: AsyncIterable | Iterable): AsyncGenerator { + for await (const entry of entries) { + yield header(entry.path, entry.size); + let written = 0; + for await (const chunk of entry.data) { + written += chunk.length; + if (written > entry.size) throw new Error(`tar: ${entry.path} is larger than its size ${entry.size}`); + yield chunk; + } + if (written !== entry.size) throw new Error(`tar: ${entry.path} is ${written} bytes, not ${entry.size}`); + const pad = (BLOCK - (entry.size % BLOCK)) % BLOCK; + if (pad > 0) yield new Uint8Array(pad); + } + yield new Uint8Array(BLOCK * 2); +} + +/** The ustar header of a regular file at `path` of `size` bytes. */ +function header(path: string, size: number): Uint8Array { + checkPath(path); + const block = new Uint8Array(BLOCK); + const bytes = new TextEncoder().encode(path); + let name = bytes; + let prefix = new Uint8Array(); + if (bytes.length > 100) { + // Split at a `/` so the name has at most 100 bytes and the prefix at most 155. + const at = path.lastIndexOf("/", path.length - 1); + const split = new TextEncoder().encode(path.slice(0, at)).length; + if (at < 0 || split > 155 || bytes.length - split - 1 > 100) throw new Error(`tar: the path ${path} is too long`); + prefix = bytes.subarray(0, split); + name = bytes.subarray(split + 1); + } + block.set(name, 0); + field(block, 100, 8, "0000644"); + field(block, 108, 8, "0000000"); + field(block, 116, 8, "0000000"); + field(block, 124, 12, size.toString(8).padStart(11, "0")); + field( + block, + 136, + 12, + Math.floor(Date.now() / 1000) + .toString(8) + .padStart(11, "0"), + ); + block.fill(0x20, 148, 156); + block[156] = 0x30; + block.set(new TextEncoder().encode("ustar\u000000"), 257); + block.set(prefix, 345); + const sum = block.reduce((total, byte) => total + byte, 0); + field(block, 148, 8, `${sum.toString(8).padStart(6, "0")}\u0000 `); + return block; +} + +/** Writes `text` (ASCII) at `offset` of `block`, NUL-terminated within `length`. */ +function field(block: Uint8Array, offset: number, length: number, text: string): void { + const bytes = new TextEncoder().encode(text); + if (bytes.length > length) throw new Error(`tar: ${text} does not fit ${length} bytes`); + block.set(bytes, offset); +} + +/** Refuses a path an archive may not hold: absolute, with `..`, `.`, an empty segment, a backslash or a NUL. */ +function checkPath(path: string): void { + const segments = path.split("/"); + const bad = path === "" || path.includes("\\") || path.includes("\u0000") || path.startsWith("/"); + if (bad || segments.some((segment) => segment === "" || segment === "." || segment === "..")) { + throw new Error(`tar: the path ${JSON.stringify(path)} is refused`); + } +} + +/** + * Reads the archive `source` entry by entry (each file's data read fully before the next entry is asked for), refusing + * what it may not hold (see the file's header). + */ +export async function* readTar(source: AsyncIterable): AsyncGenerator { + const reader = new ByteReader(source); + for (;;) { + const block = await reader.read(BLOCK); + if (block.length < BLOCK) throw new Error("tar: the archive ends inside a header"); + if (block.every((byte) => byte === 0)) return; + const { path, size, type } = parseHeader(block); + const padded = Math.ceil(size / BLOCK) * BLOCK; + if (type === "directory") { + yield { kind: "directory", path }; + await reader.skip(padded); + continue; + } + let left = size; + const data = (async function* () { + while (left > 0) { + const chunk = await reader.read(Math.min(left, 64 * 1024)); + if (chunk.length === 0) throw new Error(`tar: the archive ends inside ${path}`); + left -= chunk.length; + yield chunk; + } + })(); + yield { kind: "file", path, size, data }; + // What the consumer left unread (also when it ended the data's iteration early), then the padding. + await reader.skip(left + padded - size); + } +} + +/** A header's path, size and type, its checksum checked; anything but a regular file or a directory refused. */ +function parseHeader(block: Uint8Array): { + readonly path: string; + readonly size: number; + readonly type: "file" | "directory"; +} { + const stored = Number.parseInt(text(block, 148, 8).trim(), 8); + let sum = 0; + for (let index = 0; index < BLOCK; index++) sum += index >= 148 && index < 156 ? 0x20 : block[index]!; + if (stored !== sum) throw new Error("tar: a header's checksum does not hold"); + const flag = String.fromCharCode(block[156]!); + const type = flag === "0" || flag === "\u0000" ? "file" : flag === "5" ? "directory" : undefined; + if (type === undefined) throw new Error(`tar: an entry of type ${JSON.stringify(flag)} is refused (files only)`); + const prefix = text(block, 345, 155); + const name = text(block, 0, 100); + const path = (prefix === "" ? name : `${prefix}/${name}`).replace(/\/$/, ""); + checkPath(path); + const size = Number.parseInt(text(block, 124, 12).trim() || "0", 8); + if (!Number.isSafeInteger(size) || size < 0) throw new Error(`tar: ${path} has no valid size`); + return { path, size: type === "directory" ? 0 : size, type }; +} + +/** The NUL-terminated text of a header field. */ +function text(block: Uint8Array, offset: number, length: number): string { + const bytes = block.subarray(offset, offset + length); + const end = bytes.indexOf(0); + return new TextDecoder().decode(end < 0 ? bytes : bytes.subarray(0, end)); +} + +/** Reads exactly as many bytes as asked from a stream of chunks (fewer only at its end). */ +class ByteReader { + readonly #source: AsyncIterator; + #buffer = new Uint8Array(); + + constructor(source: AsyncIterable) { + this.#source = source[Symbol.asyncIterator](); + } + + /** Up to `length` bytes: `length` unless the stream ends first. */ + async read(length: number): Promise { + while (this.#buffer.length < length) { + const next = await this.#source.next(); + if (next.done === true) break; + const joined = new Uint8Array(this.#buffer.length + next.value.length); + joined.set(this.#buffer); + joined.set(next.value, this.#buffer.length); + this.#buffer = joined; + } + const taken = this.#buffer.subarray(0, Math.min(length, this.#buffer.length)); + this.#buffer = this.#buffer.subarray(taken.length); + return taken; + } + + /** Skips `length` bytes. */ + async skip(length: number): Promise { + let left = length; + while (left > 0) { + const skipped = await this.read(Math.min(left, 64 * 1024)); + if (skipped.length === 0) throw new Error("tar: the archive ends early"); + left -= skipped.length; + } + } +} diff --git a/packages/chord/src/next-node/app/uploads.ts b/packages/chord/src/next-node/app/uploads.ts new file mode 100644 index 000000000..6ce14e0b0 --- /dev/null +++ b/packages/chord/src/next-node/app/uploads.ts @@ -0,0 +1,172 @@ +// Chunked uploads of one `Files` object (slice 13 §4, design §23 "Chunked transfers"): each upload one file handle in +// the run's directory, written at positions. An append is accepted at the offset the accepted appends reached, so a +// writer may send several without waiting (the call window allows it): each write lands at its own position whatever +// order the thread pool completes them in. `uploaded`, `finish` and expiry wait for an upload's writes in flight; a +// failed write ends its upload. + +import type { FileHandle } from "node:fs/promises"; +import { mkdir, open, rename, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { ChordError } from "../../next/data/index.ts"; +import { invalid } from "../../next/data/internal.ts"; +import { newToken } from "./secrets.ts"; + +/** An upload under way: its file, the bytes accepted, its writes in flight, its first failed write, its end. */ +type Upload = { + readonly name: string; + readonly mime: string; + readonly handle: FileHandle; + size: number; + touched: number; + readonly writes: Set>; + failed: boolean; + ended: Promise | undefined; +}; + +/** A finished upload: its name, type and size, its bytes moved to the path `finish` was given. */ +export type Finished = { readonly name: string; readonly mime: string; readonly size: number }; + +/** + * The uploads under way of one `Files` object, in the directory `run` (this object's own, made on first use), at most + * `max` at once: each holds a file handle until it ends, so a caller beginning uploads without end would exhaust the + * process's descriptors. + */ +export class Uploads { + readonly #run: string; + readonly #max: number; + readonly #uploads = new Map(); + + constructor(run: string, max: number) { + this.#run = run; + this.#max = max; + } + + /** The bytes accepted by every upload under way (they count against the host's cap). */ + get size(): number { + let size = 0; + for (const upload of this.#uploads.values()) size += upload.size; + return size; + } + + /** A new upload of `name` and `mime`: its id; `unavailable` (`busy`) while `max` are under way. */ + async begin(name: string, mime: string): Promise { + if (this.#uploads.size >= this.#max) { + throw new ChordError({ + code: "unavailable", + reason: "busy", + message: `files: ${this.#max} uploads are under way; finish one, or wait for an idle one to expire`, + }); + } + const id = newToken(12); + await mkdir(this.#run, { recursive: true, mode: 0o700 }); + const handle = await open(join(this.#run, id), "wx", 0o600); + const upload: Upload = { + name, + mime, + handle, + size: 0, + touched: Date.now(), + writes: new Set(), + failed: false, + ended: undefined, + }; + this.#uploads.set(id, upload); + return id; + } + + /** + * Writes `bytes` at `offset` of upload `id`, which must be where the accepted appends reached (else `invalid`, its + * details the expected `{ offset }`); `room(adding, fileSize)` refuses what the caps do not allow, before anything + * is accepted. A failed write ends the upload. + */ + async append( + id: string, + offset: number, + bytes: Uint8Array, + room: (adding: number, fileSize: number) => void, + ): Promise { + const upload = this.#underway(id); + if (offset !== upload.size) { + throw invalid("args", `files: upload ${id} is at offset ${upload.size}, not ${offset}`, { + offset: upload.size, + }); + } + room(bytes.length, upload.size + bytes.length); + // Accepted now, with no await before the write starts: the next append may follow at once, its write going to + // its own position. + upload.size += bytes.length; + upload.touched = Date.now(); + const write = upload.handle.write(bytes, 0, bytes.length, offset); + upload.writes.add(write); + try { + await write; + } catch (error) { + upload.writes.delete(write); + if (!upload.failed) { + upload.failed = true; + this.#uploads.delete(id); + await this.#end(upload); + await rm(join(this.#run, id), { force: true }); + } + throw error; + } + upload.writes.delete(write); + } + + /** The bytes of upload `id` written so far, once its writes in flight landed: where a writer resumes. */ + async uploaded(id: string): Promise { + const upload = this.#underway(id); + await Promise.allSettled([...upload.writes]); + if (upload.failed) throw failedUpload(id); + return upload.size; + } + + /** Ends upload `id` once its writes landed and moves its file to `path`; refused when a write of it failed. */ + async finish(id: string, path: string): Promise { + const upload = this.#underway(id); + // Taken out first: an append arriving now finds no upload. + this.#uploads.delete(id); + await this.#end(upload); + if (upload.failed) throw failedUpload(id); + await rename(join(this.#run, id), path); + return { name: upload.name, mime: upload.mime, size: upload.size }; + } + + /** Drops the uploads unwritten for `idleMs`, once their writes landed. */ + async expire(idleMs: number): Promise { + const now = Date.now(); + for (const [id, upload] of this.#uploads) { + if (now - upload.touched >= idleMs) await this.#discard(id, upload); + } + } + + /** Drops every upload under way (their object ended). */ + async close(): Promise { + for (const [id, upload] of this.#uploads) await this.#discard(id, upload); + } + + /** The upload `id` under way; else `invalid`. */ + #underway(id: string): Upload { + const upload = this.#uploads.get(id); + if (upload === undefined) throw invalid("args", `files: no upload ${id} (finished, failed, expired or unknown)`); + return upload; + } + + /** Takes upload `id` out, closes its file once its writes landed and removes it. */ + async #discard(id: string, upload: Upload): Promise { + this.#uploads.delete(id); + await this.#end(upload); + await rm(join(this.#run, id), { force: true }); + } + + /** Closes `upload`'s file once its writes in flight landed; once, whoever ends it first. */ + #end(upload: Upload): Promise { + upload.ended ??= Promise.allSettled([...upload.writes]).then(() => upload.handle.close()); + return upload.ended; + } +} + +/** The refusal of an upload one of whose writes failed. */ +function failedUpload(id: string): Error { + return invalid("args", `files: upload ${id} failed to write and was dropped`); +} diff --git a/packages/chord/src/next-node/app/users.ts b/packages/chord/src/next-node/app/users.ts new file mode 100644 index 000000000..2461ad407 --- /dev/null +++ b/packages/chord/src/next-node/app/users.ts @@ -0,0 +1,68 @@ +// The users plugin (slice 13 §3, decision "The app kit, point 2: principals and users"): accounts keyed by the user's +// id with a profile the app describes (a schema it passes in: a Slack-like profile or just a name), kept in the +// server's store. User data never rides in the principal: a principal names the user, `Users` holds the rest. +// An exported facet, not a plugin directory: it takes the app's schema as an argument (settings are arguments), and a +// plugin directory could not import it (plugins import only the app's API). The app loads it in its root by +// re-exporting it through its API (`export const users = defineUsers(Profile)` in `/v3/server`, a one-line plugin). + +import type { JsonValue } from "../../json.ts"; +import { arr, local, map, method, nul, obj, type Schema, str, union } from "../../next/contracts/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { invalid } from "../../next/data/internal.ts"; +import { defineState } from "../../next/documents/index.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; + +/** The users plugin for profiles of schema `profile`: its contract and the facet serving it on the server. */ +export function defineUsers(profile: Schema) { + const Users = local("chord.app.users", { + version: 1, + params: {}, + members: { + /** Every user: id and profile. */ + list: arr(obj({ id: str(), profile })), + /** The profile of user `id`, or `null`. */ + get: method(str(), union([profile, nul()])), + /** Sets the profile of user `id` (a new user: an account made). */ + update: method(obj({ id: str(), profile })), + /** Removes user `id`'s account. */ + remove: method(str()), + }, + }); + // Stored as JSON: the document's types cannot follow a generic profile, the contract's members do. + const stored = profile as Schema; + const Accounts = defineState("chord.app.users", { + schema: obj({ users: map(stored) }), + version: 1, + initial: () => ({ users: {} }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 50, + }); + const facet: Facet

= defineFacet

((host) => + host.serve(Users, (_params, self) => { + const accounts = self.state(Accounts); + return { + list: self.derive([accounts], ([value]) => + Object.entries(value.users).map(([id, each]) => ({ id, profile: each as T })), + ), + // A trust boundary: `id` is the caller's (`Object.hasOwn`: no inherited member is a user). Written by + // `update` from a `T`. + get: async (id) => (Object.hasOwn(accounts.value.users, id) ? (accounts.value.users[id] as T) : null), + update: async ({ id, profile: next }, ctx) => { + // As a key it would set the record's prototype, not add a user. + if (id === "__proto__") throw invalid("args", "users: __proto__ is no user id"); + await accounts.change((draft) => { + draft.users[id] = next as JsonValue; + }, ctx); + return null; + }, + remove: async (id, ctx) => { + await accounts.change((draft) => { + delete draft.users[id]; + }, ctx); + return null; + }, + }; + }), + ); + return { Users, facet }; +} diff --git a/packages/chord/src/next-node/app/worker.ts b/packages/chord/src/next-node/app/worker.ts new file mode 100644 index 000000000..d9addad45 --- /dev/null +++ b/packages/chord/src/next-node/app/worker.ts @@ -0,0 +1,77 @@ +// `startWorker` (slice 13 §1): a worker of an app built with the kit, in its entry module. `workerMain` plus the kit's +// part: the worker's directory under its kind and address with its store, the host at the worker's address acting as +// the principal the server gave it, connected to the server first, its plugins loaded with the kit's worker facets +// (`WorkerInfo`, `Files`). + +import { join } from "node:path"; +import type { JsonValue } from "../../json.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { createHost } from "../../next/host/index.ts"; +import { openStore, sqliteBackend } from "../../next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../sqlite/node.ts"; +import type { Watchable } from "../../watch.ts"; +import { workerMain } from "../workers/index.ts"; +import type { AppApi, AppKind, KitStart } from "./app.ts"; +import { hostDataFacet } from "./data.ts"; +import { filesFacet } from "./files.ts"; +import { workerInfoFacet } from "./info.ts"; +import { privateDirectory, privateStore } from "./layout.ts"; + +/** What `startWorker` takes: the app's API and kinds as the server's `startServer` has them, this process's modules. */ +export type WorkerOptions = { + readonly api: AppApi; + readonly kinds: Readonly>; + /** Extra keep-alive counts of the app (a record of numbers, all zero when idle). */ + readonly busy?: Watchable>>; + /** The worker host's errors (`HostError`) and a failed start's. */ + readonly onError: (error: unknown) => void; +}; + +/** Runs this process as a worker of the app: once the start message arrived, its host is up and its plugins loaded. */ +export function startWorker

(options: WorkerOptions): void { + // The worker's facet kind comes with the start message: every kind's packages are offered, its own are imported. + const packages = Object.assign({}, ...Object.values(options.kinds).map((kind) => kind.packages ?? {})); + workerMain( + async ({ address, start, parent, plugins, onClose }, context) => { + // The server's `startServer` sends `{ kit, app }`. + const { kit, app } = start as { readonly kit: KitStart; readonly app: JsonValue }; + await privateDirectory(kit.dir); + await privateStore(join(kit.dir, "store.sqlite")); + const store = await openStore( + sqliteBackend(await openNodeSqliteDatabase(join(kit.dir, "store.sqlite"))), + context, + ); + onClose((ctx) => store.close(ctx)); + // Principals are the app's: the server made these with its `principal` options. + const host = createHost

({ + address, + principal: kit.principal as unknown as P, + store, + onError: options.onError, + }); + onClose((ctx) => host.close(ctx)); + // First: the server sees the worker started, then the load runs. + host.connect(parent, { principal: kit.server as unknown as P, trusted: true }, context); + const info = { + address: { kind: address.kind, key: address.key, params: address.params as JsonValue }, + start: app, + }; + const core = [ + { name: "chord.app.worker-info", facet: workerInfoFacet

(info) }, + { name: "chord.app.files", facet: filesFacet

(join(kit.dir, "files"), kit.files, kit.uploads) }, + { + name: "chord.app.data", + facet: hostDataFacet

(store, join(kit.dir, "files"), kit.server as unknown as P), + }, + ]; + await plugins.load(host, { kind: kit.facet, core }, context); + return options.busy === undefined ? {} : { busy: options.busy }; + }, + { + hostPackages: { ...options.api.modules, ...packages }, + // Worker errors are public (decision "worker errors in public"): a fixed message, never the error's own. + redact: () => ({ message: "worker failed" }), + onError: options.onError, + }, + ); +} diff --git a/packages/chord/src/next-node/http/README.md b/packages/chord/src/next-node/http/README.md new file mode 100644 index 000000000..fbb51a58e --- /dev/null +++ b/packages/chord/src/next-node/http/README.md @@ -0,0 +1,180 @@ +# http + +The HTTP component (slice 10 §2, design §23; decisions "The HTTP core is a fetch handler", "The HTTP server is a native +object", "The HTTP component's security baseline", "Slice 10 fix round, allowed hosts only through contributions", +"Settings are arguments, extensions are registries", "A CSP nonce per response"): the server's HTTP and WebSocket +entry. Plugins contribute routes, wraps, sockets, allowed hosts, framers, relays and +CSP sources to the registry `Http`; routes return response descriptions. The core is one Web-standard handler, +`fetch(Request) → Response`; Chord's own Node server (the listener) is the easy path, an app with a server of its own +mounts the handler. Node-only; `ws` is an optional peer dependency of `@earendil-works/chord`, imported only here. +Exported as `@earendil-works/chord/next/http` once v2 has its entry point. + +## For apps (`index.ts`) + +```ts +// the app's settings are arguments, fixed for the component's life (changing one is a restart) +const http = createHttp(server, { + listen: { host: "127.0.0.1", port: 8788 }, // Chord's own listener; port 0 picks one; none: no listener + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }), // or a value + opened: (request, principal) => () => linkClosed() }, // called per accepted link; its return at its close + closeTimeout: 30_000, + onError: (error) => log(error), // HttpError +}, ctx); +// the app's core facet: who someone is (a registry: plugins may add authenticators too) +const app = defineFacet((_host, self) => { + self.contribute(Authenticators, (request, ctx) => principalFromCookie(readCookie(request, "session", secret))); +}); +const core = [{ name: "chord.http", facet: http.facet }, { name: "@app/core", facet: app }]; +await plugins.load(server, { kind: "server", core, modules }, ctx); +const { port } = await http.listening(ctx); +… +await http.close(ctx); // first: drains running requests (host.close would seal their scopes at once) +await server.close(ctx); +``` + +```ts +// a plugin's server facet +self.contribute(Http, { + routes: { + "GET /notes/:id": async (request, ctx) => ({ kind: "json", value: … }), // needs a principal (else 401) + "GET /status": { public: true, handler: async () => ({ kind: "json", value: "ok" }) }, + "GET /": { page: true, handler: async (request) => ({ kind: "html", body: ``, + ``, + ``, + ].join("\n"); +} diff --git a/packages/chord/src/next-node/tui/README.md b/packages/chord/src/next-node/tui/README.md new file mode 100644 index 000000000..79803bf19 --- /dev/null +++ b/packages/chord/src/next-node/tui/README.md @@ -0,0 +1,75 @@ +# tui (launcher) + +The trusted side of a terminal client (slice 11 §4, decision "Slice 11, P8"; p2's `tui-client.ts`, `tui-sandbox.ts`, +`tui-bootstrap.mjs`, `tui-filter.ts`): `@earendil-works/chord/next/tui/launcher` once v2 has its entry point. Needs Node +26.5.1 or later (the Node its sandbox was checked on); `requireSandboxNode` refuses older ones with a clear error. + +```ts +const terminal = await startTerminal({ + url: "http://127.0.0.1:8788", + credentials: { kind: "file", path: "~/.acme/owner-token" }, // or { kind: "token", token }, { kind: "pairing", label, save? } + mode: "dark", colors: "truecolor", // the terminal's background and colours +}); +process.exitCode = await terminal.exited; +``` + +- **Credentials** (`credentials.ts`): the launcher keeps the token; pairing asks for a code, shows it, polls every 2 s + until the owner approves it, and may save the token (created, mode 0600). +- **The sandbox** (`sandbox.ts`): a Node child process of the launcher's binary, in two layers: + - the operating system's: stdin, stdout and stderr pipes to the launcher (no descriptor of the child is a terminal), + an empty environment (but `LD_LIBRARY_PATH` when the launcher has one: it needed it to run the same binary), a + private directory holding only the bootstrap; `--permission` allowing only reading the bootstrap's file (no other + file, no network, child processes, workers, addons, inspector or WASI), `--no-experimental-sqlite` (SQLite opens + files past the permission model), a 96 MiB heap. The bootstrap is a file, not `-e`, which puts every builtin on + `globalThis`; + - JavaScript's, a second layer: only the bootstrap evaluates downloaded code (the base bundle, at most 8 MiB, a + CommonJS script; the plugins' registered scripts through the bridge's `evaluate`), with `vm.Script` (options of no + prototype, no dynamic import callback: `import()` throws) and a `require` (and `process.getBuiltinModule`) + answering a short list of builtins that compute in memory (`SANDBOX_BUILTINS`, looked up in an object of no + prototype); `process` is a frozen plain object with `SANDBOX_PROCESS` only; the bootstrap keeps its own references + to what it uses, and Node's `process` has its listener calls as own properties (changing the shared + `EventEmitter.prototype` catches nothing: Opus E2); every prototype of Node's `process`, `Object.prototype` + included, takes no new property and keeps its own ones (a getter for a property Node reads on its process, such as + `noDeprecation`, would get the real one: Opus E7); + - messages both ways as length-prefixed binary frames (`pipe.ts`: a header as JSON, the text or bytes as body), each + at most 17 MiB; the launcher checks every message and ends a sandbox sending a malformed one or not reading; + - known gaps, each needing the JavaScript layer broken first unless noted: the child may signal the user's processes + (Node has no permission for signals; the launcher ignores SIGUSR1, which would open its inspector: Opus E8), read + the machine's details (`os`), and learn whether any path exists (`require.resolve` fails one way for a file that + exists, another for one that does not: Opus E13, a Node gap); memory outside its heap has no limit (only the child + dies). +- **The launcher** (`launcher.ts`): downloads `/_chord/tui/start`, the base bundle and the combined file with the token + (each body read up to 8 MiB, no further); gives the sandbox input (raw mode on a TTY) and resizes; writes its frames + through the output filter; opens and relays its link sockets to `/chord` (the token in the upgrade's header; at most + four at once); answers its fetches of terminal files only (`terminalFile`: the path resolved first, then + `/_chord/tui/` or `/_chord/tui/plugin/` of the server's origin); starts a new sandbox only when the server's + base bundle is another than the running one; prints the server's text (a pairing code, which must be a plain code) + through the filter; at its end resets the terminal's modes (sync, mouse, focus, paste, cursor, alternate screen, + colours, scroll region, autowrap), also when its process exits without that end; answers at most eight requests of + the sandbox at once (more are refused at once); ignores SIGUSR1 while it runs (no inspector opened by a signal). + Nothing the sandbox sends ends the launcher: a send on a socket not yet open is dropped, a socket with 16 MiB waiting + is closed, a message it fails on ends the client. +- **The output filter** (`filter.ts`): text and drawing sequences pass (cursor moves, erasing, SGR colours, the alternate + screen, bracketed paste, sync, mouse reporting); OSC (the clipboard among them), DCS and other control strings, + other modes and ESC sequences are dropped; an unterminated sequence ends with its frame; frames above 64 KiB are + dropped (the sandbox splits a large write into frames between sequences: `next-tui/frames.ts`). +- **Actions** (`actions.ts`), each confirmed on the launcher's prompt (the bottom lines, black on white after a reset + of all attributes, whatever the sandbox left; `y`): one at a time (a second request is refused at once); from the + request to the action's end the sandbox's frames are held (1 MiB at most); while the prompt is open its keys go to + it, and those in its first 300 ms are ignored (typed for the sandbox); a synchronized update the sandbox began is + closed and the prompt drawn last (again after a resize); its lines are wrapped to the terminal's width, never cut (a + prompt taller than the terminal is refused). Afterwards the held frames are written and the sandbox asked to redraw. + `copy` shows the text (five rows, then how many more characters) and refuses control and bidirectional characters (a + newline and a tab aside), then the launcher writes OSC 52. `open` takes http and https URLs without user info, names + the host first, and opens with `open`, `xdg-open` or `rundll32 url.dll,FileProtocolHandler` (as the coding agent), + never through a shell. `attach` refuses a path with control, format or unusual whitespace characters (on Windows + also UNC, device and `//host` paths and reserved device names), shows the path resolved by its text (its name, then + its directory) without looking at the machine, after a yes resolves its links and asks again when they lead + elsewhere, then opens it without blocking (no wait on a FIFO) and reads it only when it is the file confirmed (its + device and inode as when the last prompt was shown: no parent directory swapped for a link meanwhile) and a regular + file of at most 16 MiB, at the size its open handle had; one refusal for anything else. While a prompt is open, + Ctrl-C ends the client. +- **The way out:** three Ctrl-C within a second always end the client (exit code 130, the terminal reset), whatever the + sandbox does; each still reaches the sandbox, so an app that handles Ctrl-C itself works as before. + +Tests: `test/next-node/tui/` (the launcher and sandbox ones under `CHORD_TUI_NODE` or `node26`; skipped without). diff --git a/packages/chord/src/next-node/tui/actions.ts b/packages/chord/src/next-node/tui/actions.ts new file mode 100644 index 000000000..491a8f10d --- /dev/null +++ b/packages/chord/src/next-node/tui/actions.ts @@ -0,0 +1,212 @@ +// The launcher's actions (slice 11 §4; fix rounds 1 to 4: Fable's H1, H2, M1, L4, L6 and Opus's M1, M3, L2, L3, E5, +// E6, E9, E12, R1): what the sandbox may ask of the machine, each confirmed by the user on the launcher's own prompt +// before it touches it. Every line the prompt shows is wrapped to the terminal's width, never cut. Copying writes the +// text to the terminal's clipboard (OSC 52, which the output filter refuses to sandboxed code) and shows it first (up +// to five rows, then how many more characters); text with control characters (a newline and a tab aside) or +// bidirectional controls is refused. Opening a URL (http or https, no user info) names its host first and runs the +// platform's opener without a shell (as the coding agent's `open-browser.ts`). Attaching refuses a path with control, +// format or unusual whitespace characters (and on Windows UNC and device paths), shows the path resolved by its text +// (its name on its own line) without looking at the machine, after the user's yes resolves its links and asks again +// when they lead elsewhere, and reads the very file confirmed (its device and inode), a regular file of at most 16 MiB, +// exactly at the size its open handle has. + +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { open, realpath, stat } from "node:fs/promises"; +import { homedir } from "node:os"; +import { posix, win32 } from "node:path"; +import type { BridgeRequest } from "../../next-tui/bridge.ts"; + +/** + * The launcher's prompt on the real terminal, `columns` wide: `ask` shows `lines` (the question last) and resolves + * with the user's answer, yes or no; `write` writes to the terminal past the output filter (the launcher's own + * sequences). + */ +export type PromptTerminal = { + readonly columns: number; + ask(lines: readonly string[]): Promise; + write(data: string): void; +}; + +/** The prompt's rows for `lines` at `columns` wide: control characters removed, each line wrapped, never cut. */ +export function promptLines(lines: readonly string[], columns: number): string[] { + return lines.flatMap((line) => wrap(line, promptWidth(columns))); +} + +/** The text width of a prompt row (one column of padding each side). */ +function promptWidth(columns: number): number { + return Math.max(10, columns - 2); +} + +/** + * `line` with control characters removed, in rows of at most `width` columns, counting every character but printable + * ASCII as two (wide characters never push text past the edge, where the next row would cover it). + */ +export function wrap(line: string, width: number): string[] { + const rows: string[] = []; + let row = ""; + let used = 0; + for (const character of visible(line)) { + const cells = character >= " " && character <= "~" ? 1 : 2; + if (used + cells > width) { + rows.push(row); + row = ""; + used = 0; + } + row += character; + used += cells; + } + rows.push(row); + return rows; +} + +/** How many rows of a copied text the prompt shows before "and N more characters". */ +const SHOWN_ROWS = 5; + +/** + * Does the sandbox's `request` (`copy`, `open`) once the user confirmed it: whether it was done. Throws (the request + * refused) for text it will not copy and URLs it will not open. `opener` opens a URL (the platform's by default). + */ +export async function act( + request: Extract, + terminal: PromptTerminal, + opener: (url: string) => Promise = platformOpener, +): Promise { + if (request.kind === "copy") { + const { text } = request; + // Control characters (but newlines and tabs) and bidirectional controls could hide what a paste runs. + if (/[\u0000-\u0008\u000b-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/.test(text)) { + throw new Error("copy: the text holds control characters"); + } + const lines = text.split("\n"); + // Wrapped, never cut: a long line's end is where a pasted command would hide (Opus E6). + const rows = lines.flatMap((line) => wrap(line, promptWidth(terminal.columns) - 2)); + const hidden = rows.slice(SHOWN_ROWS).reduce((sum, row) => sum + row.length, 0); + const question = `Copy these ${text.length} characters (${lines.length} line${lines.length === 1 ? "" : "s"})? [y/N]`; + const shown = [ + ...rows.slice(0, SHOWN_ROWS).map((row) => ` ${row}`), + ...(hidden > 0 ? [` … and ${hidden} more characters`] : []), + ]; + if (!(await terminal.ask([...shown, question]))) return false; + terminal.write(`\x1b]52;c;${Buffer.from(text).toString("base64")}\x07`); + return true; + } + const url = openable(request.url); + if (url === undefined) throw new Error("open: only http and https URLs without user info open"); + if (!(await terminal.ask([` ${url.href}`, `Open ${url.host} in the browser? [y/N]`]))) return false; + await opener(url.href); + return true; +} + +/** The largest file the sandbox may be given: 16 MiB. */ +export const MAX_ATTACHMENT = 16 * 1024 * 1024; + +/** + * Reads the file at `path` (as the sandbox gave it; `~/` the home directory) for the sandbox once the user confirmed + * it: its name and bytes, or `null` when declined. Refused at once: a path with control, format or unusual whitespace + * characters, and on Windows a UNC, device or `//host` path or a reserved device name. Nothing on the machine is looked + * at before the user's yes (Opus E9: on Windows a lookup of `\\host\share` alone sends the user's credentials): the + * prompt shows the path resolved by its text (`~`, `.` and `..`); after the yes its links are resolved, and a path + * that leads elsewhere is asked about again, showing where; what is opened must be the file confirmed (its device and + * inode as when the last prompt was shown). Then it is opened without blocking (a FIFO waits for no + * writer: Opus E12) and read only when it is a regular file of at most `MAX_ATTACHMENT` bytes; one refusal for anything + * else (no hint what is there). + */ +export async function attach( + path: string, + terminal: PromptTerminal, + platform: NodeJS.Platform = process.platform, +): Promise<{ readonly name: string; readonly bytes: Uint8Array } | null> { + if (!attachable(path, platform)) throw cannotAttach(); + const paths = platform === "win32" ? win32 : posix; + const home = /^~(?:$|[\\/])/.test(path) ? paths.join(homedir(), path.slice(1)) : path; + const named = paths.resolve(home); + if (!(await terminal.ask(fileLines(named, paths, "Attach this file? [y/N]")))) return null; + const file = await realpath(named).catch(() => undefined); + if (file === undefined) throw cannotAttach(); + // The file the user confirms, by identity: what is read must be it (Opus R1: a parent directory swapped for a link + // after the yes would lead elsewhere, past `O_NOFOLLOW`, which guards the last step only). + const shown = await stat(file, { bigint: true }).catch(() => undefined); + if (shown === undefined) throw cannotAttach(); + if (file !== named && !(await terminal.ask(fileLines(file, paths, "It links here. Attach this file? [y/N]")))) { + return null; + } + // No waiting on a FIFO or device, and no link swapped in at the last step since the user saw the path. + const flags = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); + const handle = await open(file, flags).catch(() => undefined); + if (handle === undefined) throw cannotAttach(); + try { + const info = await handle.stat({ bigint: true }); + if (info.dev !== shown.dev || info.ino !== shown.ino) throw cannotAttach(); + if (!info.isFile() || info.size > BigInt(MAX_ATTACHMENT)) throw cannotAttach(); + // Exactly its size now: a file growing meanwhile is read up to what was checked. + const bytes = new Uint8Array(Number(info.size)); + let read = 0; + while (read < bytes.length) { + const { bytesRead } = await handle.read(bytes, read, bytes.length - read, read); + if (bytesRead === 0) break; + read += bytesRead; + } + return { name: paths.basename(file), bytes: bytes.subarray(0, read) }; + } finally { + await handle.close(); + } +} + +/** + * Whether `path` may be shown and, once confirmed, opened on `platform`: no character that hides or moves what the + * prompt shows (controls, format characters such as bidirectional and zero-width ones, whitespace but single spaces + * between words: Opus E5), and on Windows no UNC, device or `//host` path and no reserved device name (Opus E9). + */ +function attachable(path: string, platform: NodeJS.Platform): boolean { + if (/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]|[^\S ]| {2}|^ | $/u.test(path)) return false; + if (platform !== "win32") return true; + if (/^[\\/]{2}/.test(path)) return false; + return !path + .split(/[\\/]/) + .some((segment) => /^(?:con|prn|aux|nul|com\d|lpt\d|conin\$|conout\$)(?:\..*)?$/i.test(segment)); +} + +/** The prompt's lines for the file at `file`: its name, its directory, then `question`. */ +function fileLines(file: string, paths: typeof posix, question: string): string[] { + return [`File: ${paths.basename(file)}`, `In: ${paths.dirname(file)}`, question]; +} + +function cannotAttach(): Error { + return new Error("attach: that file cannot be attached"); +} + +/** `text` with control and format characters (bidirectional, zero width) removed: sandboxed text in the prompt. */ +function visible(text: string): string { + return text.replace(/[\p{Cc}\p{Cf}]/gu, ""); +} + +/** `url` when it is an http or https URL without user info; else `undefined`. */ +function openable(url: string): URL | undefined { + try { + const value = new URL(url); + const web = value.protocol === "http:" || value.protocol === "https:"; + return web && value.username === "" && value.password === "" ? value : undefined; + } catch { + return undefined; + } +} + +/** + * The command and arguments opening `url` on `platform`, as the coding agent opens a browser (`open-browser.ts`): never + * a shell (on Windows not `cmd /c start`, whose parsing runs what follows a `&`): `rundll32 + * url.dll,FileProtocolHandler ` on Windows, `open` on macOS, `xdg-open` elsewhere, the URL one argument. + */ +export function openerCommand(url: string, platform: NodeJS.Platform = process.platform): [string, string[]] { + if (platform === "win32") return ["rundll32", ["url.dll,FileProtocolHandler", url]]; + return [platform === "darwin" ? "open" : "xdg-open", [url]]; +} + +/** Opens `url` with the platform's opener (`openerCommand`), detached, best effort: a missing opener is no crash. */ +async function platformOpener(url: string): Promise { + const [command, args] = openerCommand(url); + // The root as its directory: no opener needs the launcher's (a WSL path too long for explorer.exe, pi #11592). + spawn(command, args, { stdio: "ignore", detached: true, cwd: "/", shell: false }) + .on("error", () => {}) + .unref(); +} diff --git a/packages/chord/src/next-node/tui/credentials.ts b/packages/chord/src/next-node/tui/credentials.ts new file mode 100644 index 000000000..35fc0b970 --- /dev/null +++ b/packages/chord/src/next-node/tui/credentials.ts @@ -0,0 +1,71 @@ +// The launcher's credentials (slice 11 §4, decision "The app kit, point 3" (c)): a device token the launcher keeps, +// never the sandbox. It is given, read from a token file (the owner token file on the server's machine, slice 13), or +// got by device pairing: the launcher asks for a code, shows it, and polls until the owner approved it in a browser; +// a paired token may be saved to a file (mode 0600) for the next start. + +import { readFile } from "node:fs/promises"; +import { writeSecret } from "../app/secrets.ts"; + +/** Where the launcher's token comes from. */ +export type TerminalCredentials = + | { readonly kind: "token"; readonly token: string } + | { readonly kind: "file"; readonly path: string } + /** Pairing with the server, the device named `label`; the token saved to `save` (created, mode 0600) when given. */ + | { readonly kind: "pairing"; readonly label: string; readonly save?: string }; + +/** How often a pairing is polled. */ +const POLL_MS = 2000; + +/** The launcher's token for the server at `url`; pairing shows its code through `say`. */ +export async function tokenOf( + credentials: TerminalCredentials, + url: URL, + say: (line: string) => void, + signal: AbortSignal, +): Promise { + switch (credentials.kind) { + case "token": + return credentials.token; + case "file": + return (await readFile(credentials.path, "utf8")).trim(); + case "pairing": + return pair(credentials, url, say, signal); + } +} + +/** Pairs this device: a code shown, the approval polled for, the token returned (and saved when asked). */ +async function pair( + credentials: Extract, + url: URL, + say: (line: string) => void, + signal: AbortSignal, +): Promise { + const asked = await post(new URL("/_chord/pair", url), { label: credentials.label }, signal); + if (!asked.ok) throw new Error(`terminal: the server refused to pair (${asked.status})`); + const { code, poll } = (await asked.json()) as { readonly code?: unknown; readonly poll?: unknown }; + // A trust boundary: the server's answer is shown on the terminal and sent back. + if (typeof code !== "string" || !/^[A-Za-z0-9-]{4,32}$/.test(code) || typeof poll !== "string") { + throw new Error("terminal: the server's pairing answer is malformed"); + } + say(`Pairing code ${code}: approve it at ${new URL(`/_chord/pair/${code}`, url)} (signed in as the owner).`); + for (;;) { + await new Promise((resolve) => setTimeout(resolve, POLL_MS)); + signal.throwIfAborted(); + const polled = await post(new URL("/_chord/pair/poll", url), { poll }, signal); + if (!polled.ok) throw new Error("terminal: the pairing ended without approval"); + const answer = (await polled.json()) as { readonly status: string; readonly token?: string }; + if (answer.status !== "approved" || answer.token === undefined) continue; + if (credentials.save !== undefined) await writeSecret(credentials.save, answer.token, "create"); + return answer.token; + } +} + +/** A JSON post. */ +function post(url: URL, body: unknown, signal: AbortSignal): Promise { + return fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal, + }); +} diff --git a/packages/chord/src/next-node/tui/filter.ts b/packages/chord/src/next-node/tui/filter.ts new file mode 100644 index 000000000..be11dcb19 --- /dev/null +++ b/packages/chord/src/next-node/tui/filter.ts @@ -0,0 +1,75 @@ +// The terminal's output filter (slice 11 §4; p2's `tui-filter.ts`): the only path from the sandbox to the real +// terminal. A frame's text and the escape sequences a terminal UI needs to draw pass (cursor moves, erasing, colours, +// the alternate screen, bracketed paste, synchronized output, mouse reporting); every other sequence is dropped, so +// sandboxed code cannot set the clipboard (OSC 52), retitle or reconfigure the terminal, or smuggle a sequence across +// frames: an unterminated one ends with its frame. + +/** The longest frame the sandbox may write. */ +export const MAX_FRAME = 64 * 1024; + +const ESC = "\x1b"; + +/** DEC private modes output may set: cursor visibility, bracketed paste, sync, alternate screen, autowrap, mouse. */ +const MODES: ReadonlySet = new Set(["25", "2004", "2026", "1049", "7", "1000", "1002", "1006", "1004"]); + +/** `output` with only the allowed sequences; throws for a frame over `MAX_FRAME`. */ +export function filterOutput(output: string): string { + if (output.length > MAX_FRAME || Buffer.byteLength(output) > MAX_FRAME) { + throw new RangeError("terminal: a frame over 64 KiB"); + } + let kept = ""; + for (let index = 0; index < output.length; index++) { + const code = output.charCodeAt(index); + if (code === 27 || (code >= 0x80 && code <= 0x9f)) { + const [next, sequence] = escapeAt(output, index); + index = next; + kept += sequence; + continue; + } + if (code === 9 || code === 10 || code === 13 || (code >= 0x20 && code !== 0x7f)) kept += output[index]; + } + return kept; +} + +/** The escape at `index`: the index of its last character, and the sequence kept (`""` when dropped). */ +function escapeAt(output: string, start: number): [number, string] { + let index = start; + const code = output.charCodeAt(index); + const introducer = code === 27 ? (output[++index] ?? "") : output[index]!; + if (introducer === "[" || code === 0x9b) { + const body = index + 1; + while ( + index + 1 < output.length && + output.charCodeAt(index + 1) >= 0x20 && + output.charCodeAt(index + 1) <= 0x3f + ) { + index++; + } + const final = output[++index] ?? ""; + const text = output.slice(body, index); + return [index, code === 27 && final !== "" && allowedCsi(text, final) ? `${ESC}[${text}${final}` : ""]; + } + if ((introducer !== "" && "]PX^_".includes(introducer)) || [0x90, 0x98, 0x9d, 0x9e, 0x9f].includes(code)) { + // A control string (OSC, DCS, ...): dropped up to its terminator, or the end of the frame. + while (++index < output.length) { + if (output.charCodeAt(index) === 0x9c || output.charCodeAt(index) === 7) break; + if (output[index] === ESC && output[index + 1] === "\\") return [index + 1, ""]; + } + return [index, ""]; + } + // Any other ESC sequence with its intermediates (charset selection, ...): dropped. + while (index < output.length && output.charCodeAt(index) >= 0x20 && output.charCodeAt(index) <= 0x2f) index++; + return [index, ""]; +} + +/** Whether the CSI sequence with parameters `body` and final `final` may reach the terminal. */ +function allowedCsi(body: string, final: string): boolean { + if (body.length > 128) return false; + if (final === "h" || final === "l") return body.startsWith("?") && MODES.has(body.slice(1)); + if (final === "m") return /^[\d;:]*$/.test(body) && !/\d{5}/.test(body); + if (!/^[\d;]*$/.test(body) || /\d{5}/.test(body)) return false; + if ("ABCDEFGHfd".includes(final)) return body.split(";").length <= ("Hf".includes(final) ? 2 : 1); + if (final === "J" || final === "K") return /^[012]?$/.test(body); + if (final === "r") return /^(?:\d{1,4};\d{1,4})?$/.test(body); + return (final === "s" || final === "u") && body === ""; +} diff --git a/packages/chord/src/next-node/tui/index.ts b/packages/chord/src/next-node/tui/index.ts new file mode 100644 index 000000000..0d0f52e82 --- /dev/null +++ b/packages/chord/src/next-node/tui/index.ts @@ -0,0 +1,8 @@ +// Chord v2's terminal launcher (`@earendil-works/chord/next/tui/launcher` once v2 has its entry point; slice 11 §4): +// the trusted Node side of a terminal client, running the app's terminal code in the sandbox. + +export type { PromptTerminal } from "./actions.ts"; +export type { TerminalCredentials } from "./credentials.ts"; +export { filterOutput } from "./filter.ts"; +export { type RunningTerminal, startTerminal, type TerminalOptions, terminalFile } from "./launcher.ts"; +export { requireSandboxNode } from "./sandbox.ts"; diff --git a/packages/chord/src/next-node/tui/launcher.ts b/packages/chord/src/next-node/tui/launcher.ts new file mode 100644 index 000000000..66a5b605b --- /dev/null +++ b/packages/chord/src/next-node/tui/launcher.ts @@ -0,0 +1,466 @@ +// `startTerminal` (slice 11 §4, decision "Slice 11, P8"): the trusted launcher of a terminal client. It signs in with +// its token (given, from a file, or by pairing), downloads the terminal kind's base bundle and plugin scripts, and +// runs them in the sandbox, always. It keeps the credentials, the real terminal and the user's choices: the sandbox +// gets input and resizes, its frames pass the output filter, its Chord link is a socket the launcher opens and relays, +// its plugin script fetches go only to the terminal surface's files, and its actions on the machine (copy, open a URL, +// attach a file) are confirmed by the launcher's own prompt. A new base bundle on the server starts a new sandbox. +// Nothing the sandbox sends can end the launcher (fix round 2, Opus E3): its messages are checked (`pipe.ts`), a send +// on a socket not yet open is dropped, every buffer and the requests in flight are bounded, a signal cannot open the +// launcher's inspector, and however the launcher's process ends the terminal is given back (fix round 3, Opus E8, +// E10, E11). During a prompt Ctrl-C ends the client: its keys are the launcher's; and three Ctrl-C within a second +// always end it (Opus R2). + +import type { Readable, Writable } from "node:stream"; +import { WebSocket } from "ws"; +import type { ColorDepth } from "../../next/surfaces/index.ts"; +import type { BridgeRequest, SandboxMessage } from "../../next-tui/bridge.ts"; +import type { TuiStart } from "../surfaces/tui.ts"; +import { act, attach, type PromptTerminal, promptLines } from "./actions.ts"; +import { type TerminalCredentials, tokenOf } from "./credentials.ts"; +import { filterOutput } from "./filter.ts"; +import { MAX_BUNDLE, requireSandboxNode, Sandbox } from "./sandbox.ts"; + +/** What `startTerminal` takes. */ +export type TerminalOptions = { + /** The app's server (`http://127.0.0.1:8788`). */ + readonly url: string; + readonly credentials: TerminalCredentials; + /** The terminal's input and output: the process's by default (raw mode while running, when a TTY). */ + readonly input?: Readable & { setRawMode?(raw: boolean): unknown; readonly isTTY?: boolean }; + readonly output?: Writable & { readonly columns?: number; readonly rows?: number }; + /** How many colours the terminal shows: true colour when `COLORTERM` says so, else 256. */ + readonly colors?: ColorDepth; + /** The terminal's background, choosing between the user's dark and light theme: dark by default. */ + readonly mode?: "dark" | "light"; + /** Opens a URL the user confirmed: the platform's opener by default. */ + readonly opener?: (url: string) => Promise; +}; + +/** A running terminal client: its exit code once it ends, and its stop. */ +export type RunningTerminal = { readonly exited: Promise; stop(): Promise }; + +/** The terminal surface's files (the server's `TUI_PREFIX`). */ +const PREFIX = "/_chord/tui/"; + +/** + * `path` resolved on the server `server` when it names one of the terminal surface's files (dot segments and encodings + * resolved first: Opus M2, Fable L1); throws otherwise: the sandbox reaches nothing else of the server through the + * launcher's token. + */ +export function terminalFile(path: unknown, server: URL): string { + const target = typeof path === "string" ? new URL(path, server) : undefined; + if ( + target === undefined || + target.origin !== server.origin || + target.search !== "" || + !TERMINAL_FILE.test(target.pathname) + ) { + throw new Error("terminal: no terminal file"); + } + return target.pathname; +} + +/** A terminal surface file's path, as the sandbox may ask for it. */ +const TERMINAL_FILE = /^\/_chord\/tui\/(?:plugin\/)?[\w.~@-]+$/; + +/** The most frames held while a prompt is open: 1 MiB (characters); later ones are dropped. */ +const MAX_HELD = 1024 * 1024; + +/** The most link sockets the sandbox may have open at once. */ +const MAX_SOCKETS = 4; + +/** The most bytes a link socket may have waiting to go out; a sandbox sending more loses the socket. */ +const MAX_SOCKET_BUFFER = 16 * 1024 * 1024; + +/** How long after a prompt appears its keys are ignored: a key typed for the sandbox does not answer it (Opus E4). */ +const PROMPT_GUARD_MS = 300; + +/** + * The terminal modes the sandbox may have set, back to the shell's: sync, mouse, focus, paste, cursor, the scroll region + * (the cursor kept where it was) and autowrap (Opus E11), screen, colours. + */ +const RESET_MODES = + "\x1b[?2026l\x1b[?1000l\x1b[?1002l\x1b[?1006l\x1b[?1004l\x1b[?2004l\x1b[?25h\x1b7\x1b[r\x1b8\x1b[?7h\x1b[?1049l\x1b[0m"; + +/** The most requests of one sandbox the launcher answers at once; more are refused at once (Opus E10). */ +const MAX_REQUESTS = 8; + +/** + * Ctrl-C: during a prompt it ends the client, so a sandbox asking again and again can always be left (Opus E11); and + * three within a second always end it, whatever the sandbox does with them (Opus R2). + */ +const INTERRUPT = "\x03"; + +/** How many Ctrl-C within `INTERRUPT_WINDOW_MS` end the client however the sandbox takes them. */ +const INTERRUPTS_TO_END = 3; +const INTERRUPT_WINDOW_MS = 1000; + +/** Starts a terminal client; rejects when it cannot sign in or download, or Node is too old for the sandbox. */ +export async function startTerminal(options: TerminalOptions): Promise { + requireSandboxNode(); + const output = options.output ?? process.stdout; + const url = new URL(options.url); + const abort = new AbortController(); + // What the launcher prints of the server's (the pairing's code) passes the output filter too. + const token = await tokenOf( + options.credentials, + url, + (line) => output.write(`${filterOutput(line)}\r\n`), + abort.signal, + ); + const launcher = new Launcher(url, token, options); + await launcher.start(); + return { exited: launcher.exited, stop: () => launcher.finish(0) }; +} + +/** One terminal client's launcher. */ +class Launcher { + readonly #url: URL; + readonly #headers: Readonly>; + readonly #options: TerminalOptions; + readonly #input: NonNullable; + readonly #output: NonNullable; + readonly #sockets = new Map(); + #sandbox: Sandbox | undefined; + /** The running base bundle's hash: an `outdated` sandbox restarts only for another. */ + #base = ""; + /** + * Whether an action is being confirmed (one at a time), from its request to its end: meanwhile the sandbox's frames + * are held (`#held`, at most `MAX_HELD` bytes), so nothing draws over the prompt, even before it appears. + */ + #confirming = false; + /** The running sandbox's requests being answered (at most `MAX_REQUESTS`). */ + #requests = 0; + /** The prompt open: its lines, its answer, and when its keys start to count. Keys go to it, not the sandbox. */ + #prompt: + | { readonly lines: readonly string[]; readonly answer: (yes: boolean) => void; readonly ready: number } + | undefined; + #held: string[] = []; + #heldBytes = 0; + #finished = false; + #exit!: (code: number) => void; + readonly exited = new Promise((resolve) => { + this.#exit = resolve; + }); + readonly #onInput = (data: Buffer | string): void => { + const text = data.toString(); + if (this.#interrupted(text)) { + void this.finish(130); + return; + } + if (this.#prompt === undefined) this.#sandbox?.post({ kind: "input", data: text }); + else if (text.includes(INTERRUPT)) void this.finish(130); + else if (Date.now() >= this.#prompt.ready) this.#answered(text === "y" || text === "Y"); + }; + readonly #onResize = (): void => { + if (this.#prompt !== undefined) this.#drawPrompt(); + this.#sandbox?.post({ kind: "resize", columns: this.#columns, rows: this.#rows }); + }; + /** When the last Ctrl-C keys came: three within a second end the client. */ + #interrupts: number[] = []; + /** The terminal given back if the launcher's process ends without `finish` (a crash). */ + readonly #onExit = (): void => { + this.#output.write(RESET_MODES); + if (this.#input.isTTY === true) this.#input.setRawMode?.(false); + }; + + constructor(url: URL, token: string, options: TerminalOptions) { + this.#url = url; + this.#headers = { authorization: `Bearer ${token}` }; + this.#options = options; + this.#input = options.input ?? process.stdin; + this.#output = options.output ?? process.stdout; + } + + /** Whether `text` brings the third Ctrl-C within a second (each still goes to the sandbox otherwise). */ + #interrupted(text: string): boolean { + const now = Date.now(); + for (const character of text) if (character === INTERRUPT) this.#interrupts.push(now); + this.#interrupts = this.#interrupts.filter((at) => now - at < INTERRUPT_WINDOW_MS); + return this.#interrupts.length >= INTERRUPTS_TO_END; + } + + // A pseudo-terminal may report 0 (Opus E4). + get #columns(): number { + return this.#output.columns || 80; + } + + get #rows(): number { + return this.#output.rows || 24; + } + + /** Downloads the base bundle and the plugins, starts the sandbox and takes the terminal. */ + async start(): Promise { + await this.#startSandbox(); + if (this.#input.isTTY === true) this.#input.setRawMode?.(true); + this.#input.on("data", this.#onInput); + this.#output.on("resize", this.#onResize); + process.once("exit", this.#onExit); + // Node opens its inspector on SIGUSR1 unless the signal has a listener: a sandbox may signal its launcher, and + // an inspector would give any local process the launcher's token and files (Opus E8). Windows has no SIGUSR1. + if (process.platform !== "win32") process.on("SIGUSR1", ignoreSignal); + } + + /** Ends the client with `code`: the terminal given back, the sockets closed, the sandbox ended. */ + async finish(code: number): Promise { + if (this.#finished) return; + this.#finished = true; + this.#input.off("data", this.#onInput); + this.#output.off("resize", this.#onResize); + process.off("exit", this.#onExit); + if (process.platform !== "win32") process.off("SIGUSR1", ignoreSignal); + if (this.#input.isTTY === true) this.#input.setRawMode?.(false); + this.#prompt?.answer(false); + this.#prompt = undefined; + this.#output.write(RESET_MODES); + this.#closeSockets(); + await this.#sandbox?.terminate(); + this.#exit(code); + } + + /** A new sandbox with what the server serves now. */ + async #startSandbox(): Promise { + const start = JSON.parse(await this.#get(`${PREFIX}start`)) as TuiStart; + const [source, combined] = await Promise.all([this.#get(start.script), this.#get(start.combined)]); + this.#base = start.base; + const colors = + this.#options.colors ?? (/truecolor|24bit/.test(process.env.COLORTERM ?? "") ? "truecolor" : "256"); + const bridge = { + base: start.base, + combined, + columns: this.#columns, + rows: this.#rows, + colors, + mode: this.#options.mode ?? "dark", + } as const; + this.#requests = 0; + this.#sandbox = await Sandbox.start( + source, + bridge, + (message) => this.#message(message), + () => void this.finish(1), + ); + } + + /** One message of the sandbox; one the launcher fails on ends the client, never the launcher's process. */ + #message(message: SandboxMessage): void { + try { + this.#handle(message); + } catch { + void this.finish(1); + } + } + + #handle(message: SandboxMessage): void { + switch (message.kind) { + case "write": + this.#write(message.data); + return; + case "socket_open": + this.#openSocket(message.socket); + return; + case "socket_send": + this.#send(message.socket, message.bytes); + return; + case "socket_close": + this.#sockets.get(message.socket)?.close(); + return; + case "request": + void this.#answer(message.id, message.request); + return; + case "outdated": + void this.#outdated(); + return; + case "failed": + this.#output.write("terminal: the terminal client failed to start\n"); + void this.finish(1); + return; + case "exit": + void this.finish(message.code); + } + } + + /** Writes the sandbox's frame through the output filter (held while a prompt is open); one over 64 KiB is dropped. */ + #write(data: string): void { + if (this.#confirming) { + if (typeof data === "string" && this.#heldBytes + data.length <= MAX_HELD) { + this.#held.push(data); + this.#heldBytes += data.length; + } + return; + } + try { + this.#output.write(filterOutput(data)); + } catch { + // Over 64 KiB (or no text): dropped, as an escape it could hide would be. + } + } + + /** Opens socket `socket` to the links endpoint with the launcher's token, relaying it to the sandbox. */ + #openSocket(socket: number): void { + // A few at a time: the connection loop needs one; a runaway sandbox gets no more. + if (this.#sockets.size >= MAX_SOCKETS) { + this.#sandbox?.post({ kind: "socket_closed", socket }); + return; + } + const target = new URL("/chord", this.#url); + target.protocol = target.protocol === "https:" ? "wss:" : "ws:"; + const ws = new WebSocket(target, { headers: this.#headers }); + this.#sockets.set(socket, ws); + ws.on("open", () => this.#sandbox?.post({ kind: "socket_opened", socket })); + ws.on("message", (data: Buffer) => { + this.#sandbox?.post({ kind: "socket_message", socket, bytes: new Uint8Array(data) }); + }); + const closed = (): void => { + if (this.#sockets.get(socket) !== ws) return; + this.#sockets.delete(socket); + this.#sandbox?.post({ kind: "socket_closed", socket }); + }; + ws.on("close", closed); + ws.on("error", closed); + } + + /** + * Sends `bytes` on socket `socket` once it is open (before, the sandbox's link holds its messages; a send then is + * dropped: Opus E3); a socket with more than 16 MiB waiting is closed. + */ + #send(socket: number, bytes: Uint8Array): void { + const ws = this.#sockets.get(socket); + if (ws?.readyState !== WebSocket.OPEN) return; + if (ws.bufferedAmount + bytes.length > MAX_SOCKET_BUFFER) ws.terminate(); + else ws.send(bytes); + } + + /** Answers the sandbox's request `id`. */ + async #answer(id: number, request: BridgeRequest): Promise { + // The sandbox that asked: a reply never goes to the one a restart started meanwhile. + const sandbox = this.#sandbox; + if (sandbox === undefined) return; + if (this.#requests >= MAX_REQUESTS) { + sandbox.post({ kind: "refused", id, message: "terminal: too many requests at once" }); + return; + } + this.#requests++; + try { + let value: unknown; + if (request.kind === "fetch") value = await this.#get(terminalFile(request.path, this.#url)); + else if (request.kind === "base") value = JSON.parse(await this.#get(`${PREFIX}base`)); + else if (request.kind === "attach") { + const file = await this.#confirm(() => attach(request.path, this.#prompter())); + if (file !== null) { + sandbox.post({ kind: "attached", id, name: file.name, bytes: file.bytes }); + return; + } + value = null; + } else value = await this.#confirm(() => act(request, this.#prompter(), this.#options.opener)); + sandbox.post({ kind: "reply", id, value }); + } catch (error) { + sandbox.post({ kind: "refused", id, message: error instanceof Error ? error.message : String(error) }); + } finally { + if (this.#sandbox === sandbox) this.#requests--; + } + } + + /** + * Runs the action `run` (it asks the user), one at a time: a second is refused while one runs. Afterwards the held + * frames are written and the sandbox asked to redraw. + */ + async #confirm(run: () => Promise): Promise { + if (this.#confirming) throw new Error("terminal: a prompt is open"); + this.#confirming = true; + try { + return await run(); + } finally { + this.#confirming = false; + const held = this.#held; + this.#held = []; + this.#heldBytes = 0; + for (const frame of held) this.#write(frame); + this.#sandbox?.post({ kind: "resize", columns: this.#columns, rows: this.#rows }); + } + } + + /** The launcher's own prompt on the real terminal, for the action being confirmed: its key not the sandbox's. */ + #prompter(): PromptTerminal { + return { + columns: this.#columns, + ask: (lines) => { + // Shown whole or not at all: a prompt is never cut. + if (promptLines(lines, this.#columns).length >= this.#rows) { + return Promise.reject(new Error("terminal: the prompt does not fit the terminal")); + } + return new Promise((answer) => { + this.#prompt = { lines, answer, ready: Date.now() + PROMPT_GUARD_MS }; + this.#drawPrompt(); + }); + }, + write: (data) => this.#output.write(data), + }; + } + + /** + * Draws the open prompt on the bottom lines after closing a synchronized update the sandbox began: its attributes + * reset and its colours explicit, black on white, so no state the sandbox left (concealed, black on black: Opus E4) + * applies to it. + */ + #drawPrompt(): void { + const lines = promptLines(this.#prompt!.lines, this.#columns); + const top = Math.max(1, this.#rows - lines.length + 1); + const drawn = lines.map((line, index) => `\x1b[${top + index};1H\x1b[0;30;47m\x1b[2K ${line} `); + this.#output.write(`\x1b[?2026l\x1b[s${drawn.join("")}\x1b[0m`); + } + + /** The prompt answered: cleared, the cursor back where the sandbox had it. */ + #answered(yes: boolean): void { + const { lines, answer } = this.#prompt!; + this.#prompt = undefined; + const rows = promptLines(lines, this.#columns); + const top = Math.max(1, this.#rows - rows.length + 1); + const cleared = rows.map((_row, index) => `\x1b[${top + index};1H\x1b[0m\x1b[2K`).join(""); + this.#output.write(`${cleared}\x1b[u`); + answer(yes); + } + + /** The sandbox says its code is outdated: a new sandbox only when the server's base bundle is another. */ + async #outdated(): Promise { + const base = JSON.parse(await this.#get(`${PREFIX}base`).catch(() => "null")) as unknown; + if (typeof base === "string" && base !== this.#base) await this.#restart(); + } + + /** The server has a new base bundle: a new sandbox with it, the old one ended. */ + async #restart(): Promise { + this.#closeSockets(); + await this.#sandbox?.terminate(); + this.#sandbox = undefined; + try { + await this.#startSandbox(); + this.#onResize(); + } catch { + await this.finish(1); + } + } + + #closeSockets(): void { + for (const ws of this.#sockets.values()) ws.close(); + this.#sockets.clear(); + } + + /** The text at `path` on the server, signed in; at most 8 MiB, read no further. */ + async #get(path: string): Promise { + const response = await fetch(new URL(path, this.#url), { headers: this.#headers }); + if (!response.ok || response.body === null) throw new Error(`terminal: GET ${path} answered ${response.status}`); + if (Number(response.headers.get("content-length") ?? 0) > MAX_BUNDLE) + throw new Error(`terminal: ${path} exceeds 8 MiB`); + const chunks: Uint8Array[] = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + if (size > MAX_BUNDLE) throw new Error(`terminal: ${path} exceeds 8 MiB`); + chunks.push(chunk); + } + return new TextDecoder().decode(Buffer.concat(chunks)); + } +} + +/** The launcher's SIGUSR1 listener: the signal does nothing. */ +function ignoreSignal(): void {} diff --git a/packages/chord/src/next-node/tui/pipe.ts b/packages/chord/src/next-node/tui/pipe.ts new file mode 100644 index 000000000..a775303a5 --- /dev/null +++ b/packages/chord/src/next-node/tui/pipe.ts @@ -0,0 +1,132 @@ +// The launcher's end of the sandbox's pipes (slice 11 §4; fix round 2: the sandbox a child process): messages as +// plain length-prefixed binary frames, not Node's IPC. A frame is a 4-byte big-endian length, then the payload: a +// 4-byte big-endian header length, the header (the message as JSON, its body field left out), and the body (the raw +// bytes of `bytes`, or the UTF-8 text of `data`). The sandbox's bootstrap has the same codec (`sandbox.ts`). What the +// sandbox sends is untrusted: each frame is bounded, and each message checked here before the launcher acts on it. + +import type { BridgeRequest, LauncherMessage, SandboxMessage } from "../../next-tui/bridge.ts"; + +/** The largest frame either side takes: 17 MiB (an attached file of 16 MiB and its header). */ +export const MAX_PIPE_FRAME = 17 * 1024 * 1024; + +/** The largest header of a frame from the sandbox: 1 MiB (a request's text or URL; frames carry their text as body). */ +const MAX_HEADER = 1024 * 1024; + +/** `message` as one frame for the sandbox. */ +export function launcherFrame(message: LauncherMessage): Buffer { + if (message.kind === "input") return frame({ kind: "input" }, Buffer.from(message.data)); + if (message.kind === "socket_message") return frame({ kind: message.kind, socket: message.socket }, message.bytes); + if (message.kind === "attached") + return frame({ kind: message.kind, id: message.id, name: message.name }, message.bytes); + return frame(message, undefined); +} + +/** The first frame: the bridge's start and the base bundle's text as body. */ +export function startFrame(start: unknown, source: string): Buffer { + return frame({ kind: "start", start }, Buffer.from(source)); +} + +/** A frame of `header` and `body`. */ +function frame(header: object, body: Uint8Array | undefined): Buffer { + const json = Buffer.from(JSON.stringify(header)); + const prefix = Buffer.allocUnsafe(8); + prefix.writeUInt32BE(4 + json.length + (body?.length ?? 0), 0); + prefix.writeUInt32BE(json.length, 4); + return Buffer.concat(body === undefined ? [prefix, json] : [prefix, json, body]); +} + +/** + * Cuts the sandbox's output into messages as chunks arrive; throws (the sandbox is ended) on a frame over + * `MAX_PIPE_FRAME`, a header over 1 MiB, or a message of no known shape. Each byte is copied in once, however small the + * chunks the sandbox writes. + */ +export class FrameReader { + #buffer = Buffer.alloc(64 * 1024); + #start = 0; + #end = 0; + + /** The messages complete with `chunk`. */ + push(chunk: Buffer): SandboxMessage[] { + this.#append(chunk); + const messages: SandboxMessage[] = []; + while (this.#end - this.#start >= 4) { + const length = this.#buffer.readUInt32BE(this.#start); + if (length < 4 || length > MAX_PIPE_FRAME) throw new Error("terminal: the sandbox sent a malformed frame"); + if (this.#end - this.#start < 4 + length) break; + // The message copies what it keeps: the buffer is reused. + messages.push(sandboxMessage(this.#buffer.subarray(this.#start + 4, this.#start + 4 + length))); + this.#start += 4 + length; + } + if (this.#start === this.#end) this.#start = this.#end = 0; + return messages; + } + + /** Copies `chunk` in after the unread bytes, moving them to the front or growing the buffer when it is full. */ + #append(chunk: Buffer): void { + const unread = this.#end - this.#start; + if (this.#end + chunk.length > this.#buffer.length) { + const needed = unread + chunk.length; + const target = + needed <= this.#buffer.length + ? this.#buffer + : Buffer.alloc(Math.max(needed, Math.min(this.#buffer.length * 2, MAX_PIPE_FRAME + 4))); + this.#buffer.copy(target, 0, this.#start, this.#end); + this.#buffer = target; + this.#start = 0; + this.#end = unread; + } + chunk.copy(this.#buffer, this.#end); + this.#end += chunk.length; + } +} + +/** + * The message in `payload` (a frame from the sandbox, without its length); throws for anything else. Trust boundary: + * the sandbox's pipe. + */ +function sandboxMessage(payload: Buffer): SandboxMessage { + const headerLength = payload.readUInt32BE(0); + if (headerLength > MAX_HEADER || 4 + headerLength > payload.length) throw malformed(); + const header = JSON.parse(payload.toString("utf8", 4, 4 + headerLength)) as Record | null; + const body = payload.subarray(4 + headerLength); + if (typeof header !== "object" || header === null) throw malformed(); + const socket = header.socket; + switch (header.kind) { + case "write": + return { kind: "write", data: body.toString("utf8") }; + case "socket_send": + if (!Number.isSafeInteger(socket)) throw malformed(); + // A copy: the payload's memory is the reader's. + return { kind: "socket_send", socket: socket as number, bytes: new Uint8Array(body) }; + case "socket_open": + case "socket_close": + if (!Number.isSafeInteger(socket)) throw malformed(); + return { kind: header.kind, socket: socket as number }; + case "request": + if (!Number.isSafeInteger(header.id)) throw malformed(); + return { kind: "request", id: header.id as number, request: bridgeRequest(header.request) }; + case "outdated": + return { kind: "outdated" }; + case "failed": + return { kind: "failed", message: String(header.message) }; + case "exit": + return { kind: "exit", code: Number.isSafeInteger(header.code) ? (header.code as number) : 1 }; + default: + throw malformed(); + } +} + +/** The request `value` names; throws for anything else. */ +function bridgeRequest(value: unknown): BridgeRequest { + const request = (typeof value === "object" && value !== null ? value : {}) as Record; + if (request.kind === "base") return { kind: "base" }; + if (request.kind === "fetch" && typeof request.path === "string") return { kind: "fetch", path: request.path }; + if (request.kind === "copy" && typeof request.text === "string") return { kind: "copy", text: request.text }; + if (request.kind === "open" && typeof request.url === "string") return { kind: "open", url: request.url }; + if (request.kind === "attach" && typeof request.path === "string") return { kind: "attach", path: request.path }; + throw malformed(); +} + +function malformed(): Error { + return new Error("terminal: the sandbox sent a malformed message"); +} diff --git a/packages/chord/src/next-node/tui/sandbox.ts b/packages/chord/src/next-node/tui/sandbox.ts new file mode 100644 index 000000000..c9b6384d9 --- /dev/null +++ b/packages/chord/src/next-node/tui/sandbox.ts @@ -0,0 +1,450 @@ +// The terminal's sandbox (slice 11 §4, decision "Slice 11, P8"; fix round 2: a child process, as the owner decided +// after the worker thread was escaped through its shared realm): a Node child process of the launcher's binary. +// - The operating system's side: its stdin, stdout and stderr are pipes to the launcher (never the terminal: no +// descriptor of the child is a TTY), its environment empty, its directory a private one holding only the bootstrap; +// Node's permission model with nothing allowed but reading the bootstrap (no other file read, no write, no network, +// no child processes, no workers, no addons, no inspector, no WASI) and `node:sqlite` removed +// (`--no-experimental-sqlite`: SQLite opens files past the permission model). An escape from the JavaScript layer +// reaches no terminal, no file, no socket, no launcher memory or token. +// - The JavaScript side, a second layer: downloaded code (the base bundle, a CommonJS script; the plugins' registered +// scripts) is only evaluated by the bootstrap (`vm.Script` with no dynamic import callback, so `import()` throws), +// with a `require` answering the allowed builtins only (`SANDBOX_BUILTINS`); `process` is a substitute with +// `SANDBOX_PROCESS` only. The bootstrap keeps its own references to what it uses before any downloaded code runs. +// Messages go both ways as length-prefixed binary frames (`pipe.ts`). Known gaps, each needing the JavaScript layer +// broken first unless noted: the child may signal processes of the same user (Node has no permission for signals; the +// launcher ignores SIGUSR1, which would open its inspector), read machine details (`os`), learn whether any path exists +// (`require.resolve` fails one way for a file that exists, another for one that does not: Opus E13), and allocate +// memory outside its heap (no limit without a shell's `ulimit`; only the child dies). Needs Node 26.5.1 or later. + +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { BridgeStart, LauncherMessage, SandboxMessage } from "../../next-tui/bridge.ts"; +import { FrameReader, launcherFrame, MAX_PIPE_FRAME, startFrame } from "./pipe.ts"; + +/** The largest base bundle a sandbox runs: 8 MiB. */ +export const MAX_BUNDLE = 8 * 1024 * 1024; + +/** + * The builtins code in the sandbox may `require`, each computation in memory without I/O, descriptors or machine + * details: bytes and text (`buffer`, `string_decoder`, `punycode`, `querystring`), events and streams in memory + * (`events`, `stream` and its submodules), formatting and type checks (`util`, `util/types`), URL and path strings + * (`url`, `path`), timers, assertions, hashing and randomness (`crypto`: no keys from files, no network) and compression + * in memory (`zlib`). Not `fs`, `tty`, `net`, `dgram`, `dns`, `http`, `https`, `http2`, `tls`, `child_process`, + * `cluster`, `worker_threads`, `sqlite`, `inspector`, `vm`, `v8`, `os`, `module`, `process`, `perf_hooks`, `wasi`, + * `repl`, `readline`, `trace_events` or anything else. + */ +export const SANDBOX_BUILTINS: readonly string[] = [ + "assert", + "assert/strict", + "buffer", + "crypto", + "events", + "path", + "path/posix", + "path/win32", + "punycode", + "querystring", + "stream", + "stream/consumers", + "stream/promises", + "stream/web", + "string_decoder", + "timers", + "timers/promises", + "url", + "util", + "util/types", + "zlib", +]; + +/** + * The sandbox's `process`: a frozen plain object in place of Node's (no event emitter, so no prototype shared with + * Node's internals leads back to the real one). Its properties, each harmless: which Node it is (`arch`, `platform`, + * `version`, `versions`, `release`), time (`nextTick`, `hrtime`, `uptime`), ending the sandbox (`exit`), the permission + * checks (`permission`), the allowed builtins (`getBuiltinModule`), fixed values (`env` empty, `argv`, `argv0`, + * `execArgv`, `execPath`, `title`, `cwd` `/`), `stdout` (to the terminal through the bridge) and `stderr` (dropped), + * and listener calls that keep nothing (`on`, `once`, `off`, `addListener`, `removeListener`, `removeAllListeners`, + * `emit`, `emitWarning`, `listenerCount`, `listeners`). Not `binding`, `_linkedBinding`, `dlopen`, `_rawDebug`, `kill`, + * `execve`, `chdir`, `abort`, `reallyExit`, `stdin`, `openStdin`, `report`, `pid`, `ppid`, `config`, the user and group + * calls, `loadEnvFile`, `moduleLoadList`, resource and handle listings or any other. + */ +export const SANDBOX_PROCESS: readonly string[] = [ + "addListener", + "arch", + "argv", + "argv0", + "cwd", + "emit", + "emitWarning", + "env", + "execArgv", + "execPath", + "exit", + "getBuiltinModule", + "hrtime", + "listenerCount", + "listeners", + "nextTick", + "off", + "on", + "once", + "permission", + "platform", + "release", + "removeAllListeners", + "removeListener", + "stderr", + "stdout", + "title", + "uptime", + "version", + "versions", +]; + +/** + * The child's Node flags for its script `script`: the permission model allowing only reading that file, no SQLite, a + * 96 MiB heap. Exported for the tests of the operating system's layer alone. + */ +export function sandboxFlags(script: string): string[] { + return ["--permission", `--allow-fs-read=${script}`, "--no-experimental-sqlite", "--max-old-space-size=96", script]; +} + +/** The bootstrap's file in the sandbox's directory: a script, not `-e`, which puts every builtin on `globalThis`. */ +export const BOOTSTRAP_FILE = "bootstrap.cjs"; + +/** The most bytes waiting for the sandbox to read them before it counts as stuck: 64 MiB. */ +const MAX_UNREAD = 64 * 1024 * 1024; + +/** Throws unless this Node is 26.5.1 or later, the Node the sandbox's permissions were checked on. */ +export function requireSandboxNode(version: string = process.versions.node): void { + const [major = 0, minor = 0, patch = 0] = version.split(".").map(Number); + if (major > 26 || (major === 26 && (minor > 5 || (minor === 5 && patch >= 1)))) return; + throw new Error(`the terminal client needs Node 26.5.1 or later for its sandbox (this is Node ${version})`); +} + +/** + * The child's environment: empty, but for `LD_LIBRARY_PATH` when the launcher has one: the launcher needed it to run + * this same binary (a Node built against libraries outside the system's paths), and the server cannot influence it. + */ +export function sandboxEnvironment(): Record { + const libraries = process.env.LD_LIBRARY_PATH; + return libraries === undefined ? {} : { LD_LIBRARY_PATH: libraries }; +} + +/** A running sandbox: messages both ways, and its end. */ +export class Sandbox { + readonly #child: ChildProcessWithoutNullStreams; + readonly #directory: string; + readonly #onMessage: (message: SandboxMessage) => void; + readonly #onEnd: (reason: string) => void; + readonly #reader = new FrameReader(); + readonly #exited: Promise; + #ended = false; + /** Whether the child has sent anything: its bootstrap was read, so the file goes. */ + #running = false; + + /** + * Starts a sandbox running the base bundle `source` (a CommonJS script) with `start` on its bridge; `onMessage` gets + * what it sends, `onEnd` its end (an exit, a crash, its memory limit, a malformed frame), once. + */ + static async start( + source: string, + start: BridgeStart, + onMessage: (message: SandboxMessage) => void, + onEnd: (reason: string) => void, + ): Promise { + requireSandboxNode(); + if (Buffer.byteLength(source) > MAX_BUNDLE) throw new Error("terminal: the base bundle exceeds 8 MiB"); + const directory = await mkdtemp(join(tmpdir(), "chord-tui-")); + await writeFile(join(directory, BOOTSTRAP_FILE), BOOTSTRAP); + const sandbox = new Sandbox(directory, onMessage, onEnd); + sandbox.#write(startFrame(start, source)); + return sandbox; + } + + private constructor( + directory: string, + onMessage: (message: SandboxMessage) => void, + onEnd: (reason: string) => void, + ) { + this.#directory = directory; + this.#onMessage = onMessage; + this.#onEnd = onEnd; + this.#child = spawn(process.execPath, sandboxFlags(join(directory, BOOTSTRAP_FILE)), { + stdio: ["pipe", "pipe", "pipe"], + env: sandboxEnvironment(), + cwd: directory, + windowsHide: true, + }); + this.#exited = new Promise((resolve) => this.#child.once("close", () => resolve())); + this.#child.stdout.on("data", (chunk: Buffer) => this.#read(chunk)); + // Dropped: the sandbox writes only through the bridge, and the terminal is the launcher's. + this.#child.stderr.resume(); + // A write after the child ended fails here; its end is reported by `close`. + this.#child.stdin.on("error", () => {}); + this.#child.on("error", (error) => this.#end(`the terminal client failed: ${error.message}`)); + this.#child.on("close", () => { + this.#end("the terminal client exited"); + void rm(this.#directory, { recursive: true, force: true }); + }); + } + + /** Sends `message` to the sandbox; a sandbox not reading what it was sent is ended. */ + post(message: LauncherMessage): void { + if (!this.#ended) this.#write(launcherFrame(message)); + } + + /** Ends the sandbox and waits for its exit. */ + async terminate(): Promise { + this.#ended = true; + this.#child.kill("SIGKILL"); + await this.#exited; + } + + #write(frame: Buffer): void { + if (this.#child.stdin.writableLength + frame.length > MAX_UNREAD + MAX_PIPE_FRAME) { + this.#fail("the terminal client stopped reading"); + return; + } + this.#child.stdin.write(frame); + } + + /** The sandbox's output: its messages, each passed on; a malformed one ends the sandbox. */ + #read(chunk: Buffer): void { + if (this.#ended) return; + if (!this.#running) { + this.#running = true; + // Read once at the start; a launcher killed later leaves at most an empty directory. + void rm(join(this.#directory, BOOTSTRAP_FILE), { force: true }); + } + let messages: SandboxMessage[]; + try { + messages = this.#reader.push(chunk); + } catch (error) { + this.#fail((error as Error).message); + return; + } + for (const message of messages) if (!this.#ended) this.#onMessage(message); + } + + #fail(reason: string): void { + this.#child.kill("SIGKILL"); + this.#end(reason); + } + + #end(reason: string): void { + if (this.#ended) return; + this.#ended = true; + this.#onEnd(reason); + } +} + +/** + * The trusted bootstrap (strict CommonJS, the child's main script). Before any downloaded code runs it keeps its own + * references to what it uses (so code that changes shared prototypes later cannot steer it), closes the file system's + * functions, shadows the real `process`'s listener calls, puts the substitute `process` in place, removes the module + * globals and leaves the bridge on `globalThis`; then it waits for the launcher's start frame and evaluates the base + * bundle through its own `require`. Frames as in `pipe.ts`. + */ +const BOOTSTRAP = `"use strict"; +const real = process; +const getBuiltin = real.getBuiltinModule; +const apply = Reflect.apply; +const uncurry = Function.prototype.call.bind(Function.prototype.bind, Function.prototype.call); +const parse = JSON.parse; +const stringify = JSON.stringify; +const freeze = Object.freeze; +const define = Object.defineProperty; +const BufferFrom = Buffer.from; +const BufferAlloc = Buffer.allocUnsafe; +const BufferConcat = Buffer.concat; +const readUInt32BE = uncurry(Buffer.prototype.readUInt32BE); +const writeUInt32BE = uncurry(Buffer.prototype.writeUInt32BE); +const bufferText = uncurry(Buffer.prototype.toString); +const subarray = uncurry(Buffer.prototype.subarray); +const copyBytes = uncurry(Buffer.prototype.copy); +const isUint8Array = (value) => value instanceof Uint8Array; +const builtin0 = (id) => apply(getBuiltin, real, [id]); +const Script = builtin0("node:vm").Script; +const runInThisContext = uncurry(Script.prototype.runInThisContext); +const nextTick = real.nextTick; +const hrtime = real.hrtime; +const hrtimeBigint = real.hrtime.bigint; +const uptime = real.uptime; +const exit = real.exit; +const input = real.stdin; +const output = real.stdout; +const writeOutput = uncurry(output.write); +const onInput = uncurry(input.on); +const MAX_FRAME = ${MAX_PIPE_FRAME}; +const deny = () => { + throw new Error("unavailable in the terminal's sandbox"); +}; + +// The file system's functions closed: the permission model guards paths, not the pipes' descriptors. +for (const facade of [builtin0("node:fs"), builtin0("node:fs/promises")]) { + for (const name of Object.keys(facade)) { + if (typeof facade[name] === "function") define(facade, name, { value: deny, writable: false, configurable: false, enumerable: true }); + } +} +const v8 = builtin0("node:v8"); +for (const name of ["writeHeapSnapshot", "getHeapSnapshot", "queryObjects", "setFlagsFromString", "takeCoverage", "startCpuProfile", "startHeapProfile"]) { + if (typeof v8[name] === "function") v8[name] = deny; +} +builtin0("node:module").syncBuiltinESMExports(); +// Node emits on its own process (warnings among them): its listener calls are its own properties, not the shared +// prototype's, which downloaded code may change. +const emitter = builtin0("node:events").EventEmitter.prototype; +for (const name of Object.getOwnPropertyNames(emitter)) { + if (name !== "constructor" && typeof emitter[name] === "function") define(real, name, { value: emitter[name], writable: false, configurable: false }); +} +// Node reads properties its process does not have (noDeprecation, throwDeprecation, traceDeprecation, +// traceProcessWarnings, channel, connected, …): such a read goes up the prototype chain, where a getter would get the +// real process as its this (Opus E7: Buffer(1) warns, and the warning reads process.noDeprecation). Every prototype of +// the chain, Object.prototype included, takes no new property and keeps its own ones, so no getter and no other +// prototype gets in. Not frozen: Node assigns over prototype fields (\`this._events ??= {}\`). +for (let prototype = Object.getPrototypeOf(real); prototype !== null; prototype = Object.getPrototypeOf(prototype)) { + for (const key of Reflect.ownKeys(prototype)) define(prototype, key, { configurable: false }); + Object.preventExtensions(prototype); +} + +// The allowed builtins, by name with or without "node:"; looked up in an object of no prototype. +const allowed = Object.create(null); +for (const name of ${JSON.stringify(SANDBOX_BUILTINS)}) allowed[name] = true; +const builtin = (id) => { + if (typeof id !== "string") throw new Error("unavailable in the terminal's sandbox"); + const name = id.length > 5 && id[0] === "n" && id[1] === "o" && id[2] === "d" && id[3] === "e" && id[4] === ":" ? id.slice(5) : id; + if (typeof name !== "string" || allowed[name] !== true) throw new Error(id + ": unavailable in the terminal's sandbox"); + return builtin0("node:" + name); +}; + +// Frames out: the message's header as JSON, its write text or socket bytes as the body. +const send = (message) => { + const kind = message.kind; + let header; + let body; + if (kind === "write") { + const data = String(message.data); + // Refused before it is copied: a huge string costs no buffer of its size. + if (data.length > MAX_FRAME) throw new RangeError("the terminal's bridge takes frames of at most 17 MiB"); + header = { kind }; + body = BufferFrom(data); + } else if (kind === "socket_send") { + if (!isUint8Array(message.bytes)) throw new TypeError("socket_send: bytes expected"); + header = { kind, socket: message.socket }; + body = message.bytes; + } else { + header = message; + } + const json = BufferFrom(stringify(header)); + const length = 4 + json.length + (body === undefined ? 0 : body.length); + if (length > MAX_FRAME) throw new RangeError("the terminal's bridge takes frames of at most 17 MiB"); + const prefix = BufferAlloc(8); + writeUInt32BE(prefix, length, 0); + writeUInt32BE(prefix, json.length, 4); + writeOutput(output, BufferConcat(body === undefined ? [prefix, json] : [prefix, json, body])); +}; + +// The substitute process. +const stdout = freeze({ isTTY: false, write: (data) => (send({ kind: "write", data: String(data) }), true) }); +const listening = (name) => [name, function () { return fake; }]; +const fake = freeze({ + arch: real.arch, + platform: real.platform, + version: real.version, + versions: freeze({ ...real.versions }), + release: freeze({ ...real.release }), + nextTick: (...args) => apply(nextTick, real, args), + hrtime: Object.assign((...args) => apply(hrtime, real, args), { bigint: () => apply(hrtimeBigint, real, []) }), + uptime: () => apply(uptime, real, []), + exit: (code) => apply(exit, real, [code]), + permission: real.permission, + env: freeze({}), + argv: freeze(["node"]), + argv0: "node", + execArgv: freeze([]), + execPath: "node", + title: "node", + cwd: () => "/", + getBuiltinModule: builtin, + stdout, + stderr: freeze({ isTTY: false, write: () => true }), + ...Object.fromEntries(["on", "once", "off", "addListener", "removeListener", "removeAllListeners"].map(listening)), + emit: () => false, + emitWarning: () => {}, + listenerCount: () => 0, + listeners: () => [], +}); +define(globalThis, "process", { value: fake, writable: false, configurable: false, enumerable: false }); +for (const name of ["BroadcastChannel", "localStorage", "sessionStorage"]) delete globalThis[name]; + +// Downloaded code runs only through here, with options of no prototype (nothing read from Object.prototype). +const evaluate = (text, filename) => { + const options = Object.create(null); + options.filename = String(filename); + return runInThisContext(new Script(String(text), options)); +}; + +// Frames in: the first is the start (the bridge's start and the base bundle), the rest the launcher's messages. +let listener; +let started = false; +// Unread bytes: one buffer, grown by doubling, each byte copied in once. +let pending = BufferAlloc(64 * 1024); +let start = 0; +let end = 0; +const received = (payload) => { + const headerLength = readUInt32BE(payload, 0); + const header = parse(bufferText(payload, "utf8", 4, 4 + headerLength)); + const body = subarray(payload, 4 + headerLength); + if (!started) { + started = true; + run(header.start, bufferText(body, "utf8")); + return; + } + if (header.kind === "input") header.data = bufferText(body, "utf8"); + else if (header.kind === "socket_message" || header.kind === "attached") header.bytes = new Uint8Array(body); + if (listener !== undefined) listener(header); +}; +onInput(input, "data", (chunk) => { + if (end + chunk.length > pending.length) { + const unread = end - start; + const target = unread + chunk.length <= pending.length ? pending : BufferAlloc(Math.max(unread + chunk.length, pending.length * 2)); + copyBytes(pending, target, 0, start, end); + pending = target; + start = 0; + end = unread; + } + copyBytes(chunk, pending, end); + end += chunk.length; + while (end - start >= 4) { + const length = readUInt32BE(pending, start); + if (length > MAX_FRAME) apply(exit, real, [70]); + if (end - start < 4 + length) break; + // A copy: the buffer is reused, and what the frame carries is kept by the sandbox. + const payload = BufferAlloc(length); + copyBytes(pending, payload, 0, start + 4, start + 4 + length); + start += 4 + length; + received(payload); + } + if (start === end) start = end = 0; +}); +onInput(input, "end", () => apply(exit, real, [0])); +const run = (start, source) => { + globalThis.chordBridge = freeze({ + start: freeze(start), + send, + onMessage: (next) => { + listener = next; + }, + evaluate, + }); + try { + const base = { exports: {} }; + evaluate("(function (require, module, exports) {" + source + "\\n})", "chord-base.js")(builtin, base, base.exports); + } catch (error) { + send({ kind: "failed", message: String(error && error.message) }); + } +}; +`; diff --git a/packages/chord/src/next-node/workers/README.md b/packages/chord/src/next-node/workers/README.md new file mode 100644 index 000000000..6098480f3 --- /dev/null +++ b/packages/chord/src/next-node/workers/README.md @@ -0,0 +1,95 @@ +# workers + +The workers component (slice 10 §3, §4, design §23, decision "Slice 10's points"): a forked worker process per address +of one host kind, started on demand by the route it owns, stopped when idle, never restarted by itself; its plugins are +built by the server and shipped. `workerMain` is the worker's side. Node-only. Exported as +`@earendil-works/chord/next/workers` once v2 has its entry point. + +## For apps (`index.ts`) + +```ts +// server.ts +const workers = createWorkers(server, Session, { + kind: "worker", entry: "./worker.ts", execArgv: ["--import", resolver], env: { … }, // bootstrap + authorize: (principal, address) => app.mayReach(principal, address), // the route's (synchronous) + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: (address, ctx) => app.startData(address, ctx), // JSON, before each start + lock: (address) => `app:${dataDir}:worker:${address.key}`, // one live worker per address + idleMs: () => app.idleMs(), // 5 min by default + plugins, + onOutput: (address, stream, chunk) => logs.append(address.key, stream, chunk), + onError: (error) => log(error), // WorkersError +}, ctx); +workers.state // { [address key]: WorkerState }, a JSON watchable +await workers.stop(Session({ id }), ctx); // suspended while held, then stopped +await workers.retry(Session({ id }), ctx); // a failed or suspended address starts again (if held) +await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, ctx); +await workers.close(ctx); + +// worker.ts +workerMain(async ({ address, start, parent, plugins, onClose }, ctx) => { + const host = createHost({ address, principal: { kind: "worker" }, store, onError }); + onClose((ctx) => host.close(ctx)); // closers run in reverse order + host.connect(parent, { principal: { kind: "server" }, trusted: true }, ctx); // first: `started` goes out now + await plugins.load(host, { kind: "worker", core: [{ name: "@app/core", facet: core(start) }] }, ctx); + return { busy: harness.activeTasks }; // extra keep-alive counts +}, { hostPackages: { "@earendil-works/chord/next": chordNext }, redact: (error) => ({ message: redacted(error) }), onError }); +``` + +- **The route:** `createWorkers` calls `host.route(Kind, …)` itself and keeps its `RouteHandle`. A failed address's + `resolve` rejects at once with its stored failure (its holders see `failed { waiting }`, nothing forks); the kernel + reports that rejection (`resolve_failed`) once per new hold, so every tab opening a failed session adds one `onError` + entry. +- **Per address:** a child's events (`started`, `ready`, `idle`, `fatal`, its exit, its link closed by this side) are + recorded on that child as they happen (a late event of an old child changes nothing); starts and stops run one after + another. A start waits for the previous child's exit and rejects with a crash recorded meanwhile; a stop records its + intent before `shutdown`, so the exit it causes is no crash. +- **A start:** the app's `start(address, ctx)` (aborted by `close`), then the fork (`serialization: "advanced"`, a + detached process group, the server's environment with `env` over it: a worker inherits every variable the server + has, secrets included, unless the app starts the server with a trimmed one; stdout and stderr piped: the stderr tail + kept, both mirrored to this process's own without `onOutput`), its IPC link made at once, the start message `{ + address, start, scripts, lock }` (`scripts`: the kind's current set, `plugins.scripts(kind)`). The child takes the + lock (waiting for an old holder without polling), makes its parent link and sends `started` when the app's host + connects; `resolve` returns the link then. The start timer (60 s) covers fork, lock and connect only. `ready` (after + the app's function returned, with the idle state and the load report) makes the worker `running`. +- **Failures stay** (R2): a crash (an exit nobody asked for), a `fatal`, a link closed by this side (the heartbeat, a + protocol error: the child is stopped, grace 5 s, then killed) or a failed start leaves the address `failed` + (`worker_failed` once, `worker_exited` for the exit) until `retry` or a rollout reaching workers. +- **Keep-alive:** the child reports when its idleness flips: `host.activity`'s `references`, `jobs` and `lingering` and + the app's `busy` (a JSON watchable of a record of numbers) all zero, no reload running; local calls send nothing. + The server stops a worker idle since `idleMs()` (checked every second, from `ready`). A stop: `shutdown`; the child + aborts its context, runs its closers in reverse order and exits (forced after 3 s); the server kills the tree after + 5 s. +- **`state`:** per address key `{ status: "starting" | "running" | "stopping", pid, startedAt }` or `{ status: + "failed", error: { message, code?, reason? }, at }`; no entry when stopped. The error is public (decision "worker + errors in public"): a `fatal` as the child's `redact` made it, else a fixed message (`worker failed to start`, + `worker exited (SIGSEGV)`, `worker exited (code 1)`); holders see the same. The full error and the stderr tail go + only to `onError` (`worker_failed { address, error, stderrTail }`). Every app hook of a start (`start`, the fork, + `lock`, `principal`) is inside that boundary; a throwing `authorize` refuses the hold (`forbidden`) and is reported + `authorize_failed`, its error private too. +- **`stop(address, ctx)`:** `RouteHandle.suspend`, then the stop: holders see `failed { waiting }` ("suspended") until + their last release; a later hold starts it on demand; `retry` lifts it at once. +- **`close(ctx)`:** starts waiting in `start(address)` are aborted, every child stopped, new starts reject + `released/closed`; the host's close (the route's context) closes it too, and no exit after that is a failure. +- **`step(name)`** (§4): clears the failures from before it and retries the held ones (listed `{ host, report }` with + their load report once ready, or `{ host, error }`), then ships the kind's current set to every worker whose last + set is older, in parallel, one reload outstanding per child: `{ host, report }`, `{ host, stopped: true }` (stopped + cleanly meanwhile) or `{ host, error }` (a crash, a failed reload). The step's context bounds only its wait (an abort + lists the waiting ones with the abort's error; a late reply still frees the child's slot). +- **The child** (`workerMain`): reads the start message, takes the lock, makes its parent link (`started` on the app's + `connect`), gives the app `plugins` (`WorkerPlugins.load(host, { kind, core })` over the shipped scripts, with the + child's own `hostPackages`), applies `reload` messages once the app's function returned (none while shutting down). + `shutdown`, the parent's disconnect, `SIGTERM` and `SIGINT` abort its context and run the closers registered so far, + also during startup. An uncaught error or a throwing function sends `fatal` with `redact(error)` and exits 1. +- `processLock(name, ctx)`: the lock (an abstract socket on Linux, a named pipe on Windows, a socket file elsewhere). + +## Examples + +`test/next/examples/`: `41-workers` (a worker started on the first hold, stopped when idle), `42-worker-crash` (a crash +stays failed, Retry brings it back), `43-rollout` (an edit rolled out to the server and its workers). + +## Inside the directory + +`control.ts` (control messages and errors as they cross), `lock.ts` (process locks), `process.ts` (one forked worker +as the server sees it), `workers.ts` (the component: the route, per-address records, idle stops, the rollout step, +the state), `main.ts` (`workerMain`). diff --git a/packages/chord/src/next-node/workers/control.ts b/packages/chord/src/next-node/workers/control.ts new file mode 100644 index 000000000..34eb0a66d --- /dev/null +++ b/packages/chord/src/next-node/workers/control.ts @@ -0,0 +1,121 @@ +// The control channel between the server and a worker (slice 10 §3): JSON messages beside the link's `Uint8Array` +// frames on one IPC channel (the IPC link skips them), and errors as they cross it (`{ message, code?, reason? }`, +// never a stack or a cause). + +import type { JsonValue } from "../../json.ts"; +import type { Address } from "../../next/contracts/index.ts"; +import { ChordError, type FailureCause, isChordError } from "../../next/data/index.ts"; +import { chordErrorFromJson } from "../../next/data/internal.ts"; +import type { ReloadReport } from "../../next/host/index.ts"; +import type { PluginScript } from "../plugins/runtime.ts"; + +/** An error as it crosses: its message, and a ChordError's code and reason. */ +export type CrossedError = { readonly message: string; readonly code?: string; readonly reason?: string }; + +/** A reload report as it crosses: its errors crossed. */ +export type CrossedReport = { + readonly plugins: ReloadReport["plugins"]; + readonly failed: readonly { readonly plugin: string; readonly error: CrossedError }[]; + readonly objects: readonly { readonly object: string; readonly kind: string; readonly cause?: CrossedCause }[]; +}; + +/** A failure cause as it crosses. */ +type CrossedCause = { readonly kind: FailureCause["kind"]; readonly error: CrossedError }; + +/** What the server sends a worker. */ +export type ToWorker = + | { + readonly kind: "start"; + readonly address: Address; + readonly start: JsonValue; + readonly scripts: readonly PluginScript[]; + readonly lock: string; + } + | { readonly kind: "shutdown" } + | { readonly kind: "reload"; readonly scripts: readonly PluginScript[] }; + +/** What a worker sends the server. */ +export type FromWorker = + | { readonly kind: "started" } + | { readonly kind: "ready"; readonly idle: boolean; readonly report: CrossedReport } + | { readonly kind: "idle"; readonly idle: boolean } + | { readonly kind: "fatal"; readonly error: CrossedError } + | { readonly kind: "reloaded"; readonly report: CrossedReport } + | { readonly kind: "reloaded"; readonly error: CrossedError }; + +/** `message` in its envelope, as it goes over the IPC channel. */ +export function envelope(message: ToWorker | FromWorker): { readonly [ENVELOPE]: ToWorker | FromWorker } { + return { [ENVELOPE]: message }; +} + +/** The envelope's key: a control message is `{ "chord.workers": message }`. */ +const ENVELOPE = "chord.workers"; + +/** + * The control message in an IPC message, or `undefined` for anything else (a link frame, the app's own). A trust + * boundary in name only: both ends are the app's processes, so only the envelope is checked. + */ +export function controlOf(message: unknown): M | undefined { + if (typeof message !== "object" || message === null || !(ENVELOPE in message)) return undefined; + // Sent by the other end of this channel through `envelope`. + return (message as { readonly [ENVELOPE]: M })[ENVELOPE]; +} + +/** `error` as it crosses. */ +export function crossed(error: unknown): CrossedError { + if (isChordError(error)) { + return { + message: error.message, + code: error.code, + ...(error.reason === undefined ? {} : { reason: error.reason }), + }; + } + return { message: error instanceof Error ? error.message : String(error) }; +} + +/** A crossed error as an error again: a ChordError when its code is one, an `Error` otherwise. */ +export function uncrossed(error: CrossedError): Error { + if (error.code === undefined) return new Error(error.message); + try { + return chordErrorFromJson({ + code: error.code, + message: error.message, + ...(error.reason === undefined ? {} : { reason: error.reason }), + }); + } catch { + return new Error(error.message); + } +} + +/** `report` as it crosses. */ +export function crossedReport(report: ReloadReport): CrossedReport { + return { + plugins: report.plugins, + failed: report.failed.map(({ plugin, error }) => ({ plugin, error: crossed(error) })), + objects: report.objects.map((each) => + each.kind === "parked" + ? { + object: each.object, + kind: each.kind, + cause: { kind: each.cause.kind, error: crossed(each.cause.error) }, + } + : each, + ), + }; +} + +/** A crossed report as a report again. */ +export function uncrossedReport(report: CrossedReport): ReloadReport { + return { + plugins: report.plugins, + failed: report.failed.map(({ plugin, error }) => ({ plugin, error: uncrossed(error) })), + objects: report.objects.map(({ object, kind, cause }) => { + if (cause === undefined) + return { object, kind: kind as Exclude }; + const error = uncrossed(cause.error); + // A parked object's cause is a ChordError on the worker; one that crossed without a code is wrapped as one. + const chord = isChordError(error) ? error : new ChordError({ code: "unavailable", message: error.message }); + return { object, kind: "parked", cause: { kind: cause.kind, error: chord } as FailureCause }; + }), + }; +} diff --git a/packages/chord/src/next-node/workers/index.ts b/packages/chord/src/next-node/workers/index.ts new file mode 100644 index 000000000..219d8890f --- /dev/null +++ b/packages/chord/src/next-node/workers/index.ts @@ -0,0 +1,20 @@ +// Chord v2's workers component (`@earendil-works/chord/next/workers`, slice 10 §3, §4): a forked worker process per +// address of a host kind, started on demand, stopped when idle, never restarted by itself, its plugins shipped from the +// server; `workerMain` is the worker's side. Node-only. + +export type { CrossedError } from "./control.ts"; +export { type ProcessLock, processLock } from "./lock.ts"; +export { + type WorkerMainOptions, + type WorkerPlugins, + type WorkerResult, + type WorkerStart, + workerMain, +} from "./main.ts"; +export { + createWorkers, + type WorkerState, + type Workers, + type WorkersError, + type WorkersOptions, +} from "./workers.ts"; diff --git a/packages/chord/src/next-node/workers/lock.ts b/packages/chord/src/next-node/workers/lock.ts new file mode 100644 index 000000000..279bca095 --- /dev/null +++ b/packages/chord/src/next-node/workers/lock.ts @@ -0,0 +1,142 @@ +// Process locks (slice 10 §3, Chord v1's `processLock`): one live worker per address. A lock is a listening socket the +// kernel owns for the holder's lifetime (an abstract socket on Linux and Android, a named pipe on Windows, a socket file +// elsewhere), so there is no race between two would-be holders; a socket file a crashed holder left fails closed (the +// error names it: a takeover would race another taker). A child finding the lock held waits +// without polling: it connects to the holder's socket, which the holder never answers, and tries again when that +// connection closes (the holder exited). + +import { createHash } from "node:crypto"; +import { unlinkSync } from "node:fs"; +import { lstat, mkdir } from "node:fs/promises"; +import { connect, createServer, type Server, type Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Context } from "../../context/index.ts"; + +/** A held lock: `release` frees it (the process's exit does too). */ +export type ProcessLock = { release(): Promise }; + +/** Where a lock lives: an abstract socket (Linux, Android), a named pipe (Windows) or a socket file (elsewhere). */ +export type LockAddress = { readonly kind: "abstract" | "pipe" | "file"; readonly address: string }; + +/** + * Takes the lock `name`, waiting while another process holds it (until `context` aborts: rejects with its reason). + * Throws what listening throws besides "in use", and for a stale socket file (macOS, the BSDs) an error naming it. + */ +export async function processLock(name: string, context: Context): Promise { + const key = createHash("sha256").update(name).digest("hex").slice(0, 32); + if (process.platform === "win32") + return lockAt({ kind: "pipe", address: `\\\\.\\pipe\\chord-lock-${key}` }, context); + if (process.platform === "linux" || process.platform === "android") { + return lockAt({ kind: "abstract", address: `\0chord-lock-${key}` }, context); + } + return lockAt({ kind: "file", address: await socketPath(`chord-lock-${key}.sock`) }, context); +} + +/** + * Takes the lock at `lock`, waiting while it is held. A socket file nobody accepts on (a crashed holder's) fails closed: + * taking it over races another taker (two could both unlink and listen), so the error names the file to remove. + */ +export async function lockAt(lock: LockAddress, context: Context): Promise { + const { address } = lock; + const file = lock.kind === "file"; + // Connections are kept open, never answered: a waiter learns of the release when its connection closes. + const waiters = new Set(); + const server = createServer((socket) => { + waiters.add(socket); + socket.on("error", () => {}); + socket.once("close", () => waiters.delete(socket)); + }); + for (;;) { + context.abortSignal?.throwIfAborted(); + if (await listened(server, address)) break; + const held = await waitWhileHeld(address, context); + if (!held && file) { + throw new Error(`lock: ${address} is a stale lock file (no process holds it); remove it if no worker runs`); + } + } + const removeFile = (): void => { + try { + unlinkSync(address); + } catch { + // Gone already. + } + }; + if (file) process.once("exit", removeFile); + return { + release: () => + new Promise((resolve, reject) => { + if (file) process.removeListener("exit", removeFile); + server.close((error) => (error ? reject(error) : resolve())); + // Waiters see their connections close (and `close` waits for them). + for (const socket of waiters) socket.destroy(); + }), + }; +} + +/** Whether `server` now listens on `address`; `false` when it is in use. */ +function listened(server: Server, address: string): Promise { + return new Promise((resolve, reject) => { + const failed = (error: NodeJS.ErrnoException): void => { + if (error.code === "EADDRINUSE") resolve(false); + else reject(error); + }; + server.once("error", failed); + server.listen(address, () => { + server.removeListener("error", failed); + resolve(true); + }); + }); +} + +/** + * Connects to the holder of `address` and resolves when that connection closes or the address is gone (`true`: try + * again), or with `false` when nobody accepts on it (a leftover socket file); `context`'s abort rejects. + */ +function waitWhileHeld(address: string, context: Context): Promise { + return new Promise((resolve, reject) => { + const signal = context.abortSignal; + if (signal?.aborted) { + reject(signal.reason); + return; + } + const socket = connect(address); + const abort = (): void => { + socket.destroy(); + reject(signal?.reason); + }; + signal?.addEventListener("abort", abort, { once: true }); + const settle = (held: boolean): void => { + signal?.removeEventListener("abort", abort); + resolve(held); + }; + let connected = false; + socket.once("connect", () => { + connected = true; + }); + socket.on("error", (error: NodeJS.ErrnoException) => { + if (!connected && error.code === "ECONNREFUSED") settle(false); + }); + socket.once("close", () => settle(true)); + }); +} + +/** sun_path holds 104 bytes on macOS and the BSDs, the terminating NUL included. */ +const MAX_SOCKET_PATH_BYTES = 103; + +/** + * A lock socket's path: in the temporary directory, or, when that does not fit in sun_path (a deep TMPDIR), in a + * private per-user directory under /tmp that only its owner can reach. + */ +async function socketPath(file: string): Promise { + const preferred = join(tmpdir(), file); + if (Buffer.byteLength(preferred) <= MAX_SOCKET_PATH_BYTES) return preferred; + const uid = process.getuid?.() ?? 0; + const directory = `/tmp/chord-lock-${uid}`; + await mkdir(directory, { mode: 0o700, recursive: true }); + const stats = await lstat(directory); + if (!stats.isDirectory() || stats.uid !== uid || (stats.mode & 0o077) !== 0) { + throw new Error(`lock directory ${directory} must be a directory only its owner can access`); + } + return join(directory, file); +} diff --git a/packages/chord/src/next-node/workers/main.ts b/packages/chord/src/next-node/workers/main.ts new file mode 100644 index 000000000..95d63f73a --- /dev/null +++ b/packages/chord/src/next-node/workers/main.ts @@ -0,0 +1,293 @@ +// The worker's side (slice 10 §3): `workerMain` reads the start message, takes the address's lock, makes the link to +// the server (`started` goes out when the app's host listens on it), runs the app's function with a plugin runtime over +// the shipped scripts, reports `ready` and each idle flip, applies reloads, and on `shutdown`, the parent's disconnect, +// `SIGTERM` or `SIGINT` aborts its context, runs the app's closers in reverse order and exits. + +import { BACKGROUND_CONTEXT, type Context, withAbortSignal } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Address } from "../../next/contracts/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import type { Host, LoadReport, PluginFacet, ReloadReport } from "../../next/host/index.ts"; +import { type IpcPort, ipcLink, type Link } from "../../next/link/index.ts"; +import type { Watchable } from "../../watch.ts"; +import type { HostPackages } from "../plugins/evaluate.ts"; +import { createPluginRuntime, type PluginScript } from "../plugins/runtime.ts"; +import { + type CrossedError, + controlOf, + crossed, + crossedReport, + envelope, + type FromWorker, + type ToWorker, +} from "./control.ts"; +import { processLock } from "./lock.ts"; + +/** What the app's function gets. */ +export type WorkerStart = { + readonly address: Address; + /** The start data the server's `start(address)` gave. */ + readonly start: JsonValue; + /** The link to the server: `connect` it before anything slow (`started` goes out then). */ + readonly parent: Link; + readonly plugins: WorkerPlugins; + /** Registers a closer: closers run in reverse order at shutdown. */ + onClose(closer: (context: Context) => Promise | void): void; +}; + +/** The worker's plugins: the shipped scripts of its kind, loaded on its host; the server's reloads applied to it. */ +export interface WorkerPlugins { + /** Loads `host` with the start message's scripts of `kind` and `core` (the app's own facets). */ + load

( + host: Host

, + options: { readonly kind: string; readonly core: readonly PluginFacet

[] }, + context: Context, + ): Promise; +} + +/** What the app's function returns: extra keep-alive counts (busy while any is nonzero). */ +export type WorkerResult = { readonly busy?: Watchable>> }; + +/** What `workerMain` takes besides the app's function. */ +export type WorkerMainOptions = { + /** The host packages by specifier: this process's own instances (functions cannot cross IPC). */ + readonly hostPackages: HostPackages; + /** What crosses as `fatal` for an error (never the original message unless this returns it). */ + readonly redact: (error: unknown) => CrossedError; + readonly onError: (error: unknown) => void; +}; + +/** + * Runs this process as a worker: `fn` once the start message arrived and the lock is taken. An uncaught error or a + * throwing `fn` sends `fatal` (redacted) and exits. + */ +export function workerMain( + fn: (start: WorkerStart, context: Context) => Promise, + options: WorkerMainOptions, +): void { + // `process` is the IPC port of a forked child. + new Worker(process as unknown as IpcPort, options).run(fn); +} + +/** How long a shutdown's closers get before the worker exits anyway: 3 s. */ +const FORCED_EXIT = 3_000; + +class Worker { + readonly #port: IpcPort; + readonly #options: WorkerMainOptions; + readonly #controller = new AbortController(); + readonly #context: Context; + readonly #closers: ((context: Context) => Promise | void)[] = []; + readonly #start = Promise.withResolvers>(); + /** Reloads that arrived, applied one after another once `fn` returned. */ + readonly #reloads: (readonly PluginScript[])[] = []; + #loaded: { readonly host: Host; readonly runtime: ReturnType } | undefined; + #ready = false; + #reloading = false; + #busy: Watchable>> | undefined; + /** The last idle state sent. */ + #idle: boolean | undefined; + #stopping = false; + + constructor(port: IpcPort, options: WorkerMainOptions) { + this.#port = port; + this.#options = options; + this.#context = withAbortSignal(this.#controller.signal, BACKGROUND_CONTEXT); + port.on("message", (message) => this.#message(controlOf(message))); + port.on("disconnect", () => void this.#shutdown()); + for (const signal of ["SIGTERM", "SIGINT"] as const) process.once(signal, () => void this.#shutdown()); + process.once("uncaughtException", (error) => this.#fatal(error)); + process.once("unhandledRejection", (error) => this.#fatal(error)); + } + + /** Takes the lock, makes the parent link, runs `fn`, then reports `ready` and applies reloads. */ + run(fn: (start: WorkerStart, context: Context) => Promise): void { + // A throw while shutting down is the shutdown's own cancellation (its `ctx` aborted): no fatal, the closers run. + void this.#run(fn).catch((error: unknown) => { + if (!this.#stopping) this.#fatal(error); + }); + } + + async #run(fn: (start: WorkerStart, context: Context) => Promise): Promise { + const message = await this.#start.promise; + await processLock(message.lock, this.#context); + const parent = new ParentLink(ipcLink(this.#port), () => this.#send({ kind: "started" })); + const runtime = createPluginRuntime(); + const plugins: WorkerPlugins = { + load: async (host, { kind, core }, context) => { + this.#loaded = { host: host as unknown as Host, runtime }; + return runtime.load( + host, + { kind, core, scripts: message.scripts, modules: this.#options.hostPackages }, + context, + ); + }, + }; + let report: LoadReport = { failed: [] }; + const loading: WorkerPlugins = { + load: async (host, options, context) => { + report = await plugins.load(host, options, context); + return report; + }, + }; + const start: WorkerStart = { + address: message.address, + start: message.start, + parent, + plugins: loading, + onClose: (closer) => void this.#closers.push(closer), + }; + const result = await fn(start, this.#context); + if (this.#stopping) return; + this.#busy = result?.busy; + await this.#follow(); + this.#ready = true; + const failed = new Set(report.failed.map(({ plugin }) => plugin)); + const added = message.scripts.map(({ plugin }) => plugin.name).filter((name) => !failed.has(name)); + const loaded: ReloadReport = { plugins: { added, changed: [], removed: [] }, failed: report.failed, objects: [] }; + this.#idle = this.#isIdle(); + this.#send({ kind: "ready", idle: this.#idle, report: crossedReport(loaded) }); + void this.#applyReloads(); + } + + /** Follows the host's activity and the app's `busy`: each idle flip is reported. */ + async #follow(): Promise { + const sources: Watchable[] = []; + if (this.#loaded !== undefined) sources.push(this.#loaded.host.activity); + if (this.#busy !== undefined) sources.push(this.#busy); + for (const source of sources) { + const watch = await source.watch(this.#context); + watch.start(async () => this.#flip()); + } + } + + /** Whether every count is zero: the host's references, jobs and lingering objects, the app's `busy`, no reload. */ + #isIdle(): boolean { + if (this.#reloading) return false; + const activity = this.#loaded?.host.activity.value; + if (activity !== undefined && activity.references + activity.jobs + activity.lingering > 0) return false; + return Object.values(this.#busy?.value ?? {}).every((count) => count === 0); + } + + /** Reports the idle state when it flipped (after `ready`). */ + #flip(): void { + if (!this.#ready) return; + const idle = this.#isIdle(); + if (idle === this.#idle) return; + this.#idle = idle; + this.#send({ kind: "idle", idle }); + } + + /** A control message from the server. */ + #message(message: ToWorker | undefined): void { + if (message === undefined) return; + switch (message.kind) { + case "start": + this.#start.resolve(message); + return; + case "shutdown": + void this.#shutdown(); + return; + case "reload": + // A worker shutting down applies no reload (its exit answers it). + if (this.#stopping) return; + this.#reloads.push(message.scripts); + if (this.#ready) void this.#applyReloads(); + } + } + + /** Applies the reloads that arrived, one after another, each answered `reloaded`. */ + async #applyReloads(): Promise { + if (this.#reloading || this.#reloads.length === 0) return; + const loaded = this.#loaded; + this.#reloading = true; + this.#flip(); + for ( + let scripts = this.#reloads.shift(); + scripts !== undefined && !this.#stopping; + scripts = this.#reloads.shift() + ) { + if (loaded === undefined) { + this.#send({ kind: "reloaded", error: { message: "worker: plugins.load was never called" } }); + continue; + } + try { + const report = await loaded.runtime.reload(loaded.host, scripts, this.#context); + this.#send({ kind: "reloaded", report: crossedReport(report) }); + } catch (error) { + this.#send({ kind: "reloaded", error: crossed(error) }); + } + } + this.#reloading = false; + this.#flip(); + } + + /** Aborts the context, runs the closers in reverse order and exits (forced after 3 s). */ + async #shutdown(): Promise { + if (this.#stopping) return; + this.#stopping = true; + this.#controller.abort(new Error("worker: shutting down")); + setTimeout(() => process.exit(0), FORCED_EXIT).unref(); + const context = BACKGROUND_CONTEXT; + for (const closer of this.#closers.reverse()) { + try { + await closer(context); + } catch (error) { + this.#options.onError(error); + } + } + process.exit(0); + } + + /** Sends `fatal` with the redacted error, then exits. */ + #fatal(error: unknown): void { + this.#options.onError(error); + let redacted: CrossedError; + try { + redacted = this.#options.redact(error); + } catch { + redacted = { message: "worker: fatal error" }; + } + if (!this.#port.connected) process.exit(1); + // `process.send` takes a callback once the message is out. + (process.send as (message: unknown, callback: () => void) => void)( + envelope({ kind: "fatal", error: redacted }), + () => process.exit(1), + ); + } + + /** Sends a control message; dropped when the server is gone. */ + #send(message: FromWorker): void { + if (this.#port.connected) process.send?.(envelope(message), undefined, undefined, () => {}); + } +} + +/** The link to the server as the app gets it: `started` goes out when the app's host sets its listener (`connect`). */ +class ParentLink implements Link { + readonly #inner: Link; + #started: (() => void) | undefined; + + constructor(inner: Link, started: () => void) { + this.#inner = inner; + this.#started = started; + } + + send(message: Uint8Array): void { + this.#inner.send(message); + } + + onMessage(listener: (message: Uint8Array) => void): void { + this.#inner.onMessage(listener); + const started = this.#started; + this.#started = undefined; + started?.(); + } + + close(): void { + this.#inner.close(); + } + + onClose(listener: () => void): void { + this.#inner.onClose(listener); + } +} diff --git a/packages/chord/src/next-node/workers/process.ts b/packages/chord/src/next-node/workers/process.ts new file mode 100644 index 000000000..5ed995aef --- /dev/null +++ b/packages/chord/src/next-node/workers/process.ts @@ -0,0 +1,150 @@ +// A worker process as the server sees it (slice 10 §3, Chord v1's `forkDriver`): forked with V8 serialization as a +// detached process group (the whole tree can be killed), stdout and stderr piped (the stderr tail kept, both mirrored to +// the server's own when the app takes no output), its IPC link made at the fork so the child's `hello` and early holds +// are queued, never lost; control messages, the link's close and the exit are told as they happen. + +import { type ChildProcess, fork } from "node:child_process"; +import type { Link } from "../../next/link/index.ts"; +import { ipcLink } from "../../next/link/index.ts"; +import { controlOf, envelope, type FromWorker, type ToWorker } from "./control.ts"; + +/** How a worker process is started: the bootstrap options of the workers component. */ +export type ForkOptions = { + readonly entry: string; + readonly execArgv?: readonly string[]; + readonly env?: Readonly>; + /** Its output; without it, mirrored to this process's own. */ + readonly onOutput?: (stream: "stdout" | "stderr", chunk: Uint8Array) => void; +}; + +/** How a worker process ended. */ +export type Exit = { readonly code: number | null; readonly signal: NodeJS.Signals | null }; + +/** What a worker process tells as it happens. */ +export type ProcessEvents = { + readonly control: (message: FromWorker) => void; + /** Its link was closed by this side (the heartbeat, a protocol error) while the child may live. */ + readonly linkLost: () => void; + /** Its link was closed by the child's side (it exits, or it closed its IPC channel and may live on). */ + readonly linkClosed: () => void; + readonly exited: (exit: Exit) => void; +}; + +/** How much of a worker's stderr is kept for its failure: 4 KiB. */ +const STDERR_TAIL = 4096; + +/** A control message to a child that is gone: dropped (its exit or link close tells). */ +const ignoreSendError = (): void => {}; + +/** One forked worker process. */ +export class WorkerProcess { + readonly pid: number; + readonly startedAt = Date.now(); + /** The link to the child's host, made at the fork. */ + readonly link: Link; + /** Resolves with its exit; every process it spawned is killed then. */ + readonly exited: Promise; + readonly #child: ChildProcess; + #stderr = ""; + + constructor(options: ForkOptions, events: ProcessEvents) { + const child = fork(options.entry, [], { + env: { ...process.env, ...options.env }, + execArgv: [...(options.execArgv ?? [])], + detached: true, + serialization: "advanced", + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + this.#child = child; + this.pid = child.pid ?? -1; + for (const stream of ["stdout", "stderr"] as const) { + child[stream]?.on("data", (chunk: Buffer) => { + if (stream === "stderr") this.#stderr = (this.#stderr + chunk.toString("utf8")).slice(-STDERR_TAIL); + if (options.onOutput !== undefined) options.onOutput(stream, chunk); + else process[stream].write(chunk); + }); + } + this.link = new TellingLink(ipcLink(child), events); + child.on("message", (message: unknown) => { + const control = controlOf(message); + if (control !== undefined) events.control(control); + }); + this.exited = new Promise((resolve) => { + const ended = (exit: Exit): void => { + // However the leader ended, nothing it spawned outlives it. + this.#killGroup(); + resolve(exit); + events.exited(exit); + }; + child.once("exit", (code, signal) => ended({ code, signal })); + // A fork that failed (no `exit` follows). + child.once("error", () => { + if (child.pid === undefined) ended({ code: null, signal: null }); + }); + }); + } + + /** The last bytes it wrote to stderr, if any. */ + get stderrTail(): string | undefined { + return this.#stderr === "" ? undefined : this.#stderr; + } + + /** Sends a control message; dropped when the child is gone (its exit tells). */ + send(message: ToWorker): void { + if (this.#child.connected) this.#child.send(envelope(message), ignoreSendError); + } + + /** Kills the child and every process it spawned. */ + kill(): void { + if (!this.#killGroup()) this.#child.kill("SIGKILL"); + } + + /** Kills the process group; `false` when there is none. */ + #killGroup(): boolean { + try { + process.kill(-this.pid, "SIGKILL"); + return true; + } catch { + return false; + } + } +} + +/** A link that also tells who closed it (`linkLost`: this side, `linkClosed`: the child's), whoever listens to it. */ +class TellingLink implements Link { + readonly #inner: Link; + readonly #events: Pick; + #listener: (() => void) | undefined; + #closed = false; + + constructor(inner: Link, events: Pick) { + this.#inner = inner; + this.#events = events; + inner.onClose(() => { + const here = this.#closed; + this.#closed = true; + this.#listener?.(); + if (!here) events.linkClosed(); + }); + } + + send(message: Uint8Array): void { + this.#inner.send(message); + } + + onMessage(listener: (message: Uint8Array) => void): void { + this.#inner.onMessage(listener); + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + this.#events.linkLost(); + this.#inner.close(); + } + + onClose(listener: () => void): void { + if (this.#closed) listener(); + else this.#listener = listener; + } +} diff --git a/packages/chord/src/next-node/workers/workers.ts b/packages/chord/src/next-node/workers/workers.ts new file mode 100644 index 000000000..7ba44c62a --- /dev/null +++ b/packages/chord/src/next-node/workers/workers.ts @@ -0,0 +1,742 @@ +// The workers component (slice 10 §3, §4; decision "Slice 10's points"): a forked worker process per address of one +// host kind, started on demand by the route it owns, stopped when idle, never restarted by itself (R2): a crash, a +// `fatal`, a link lost while its child lives or a failed start leaves the address failed until `retry` or a rollout. +// Per address, a child's events are recorded on that child as they happen; starts and stops run one after another. + +import type { Context } from "../../context/index.ts"; +import { awaitWithContext, BACKGROUND_CONTEXT, withAbortSignal } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import type { Address, HostKind, KindShape } from "../../next/contracts/index.ts"; +import type { Params } from "../../next/contracts/internal.ts"; +import { ChordError, type Principal } from "../../next/data/index.ts"; +import type { Host, RouteHandle } from "../../next/host/index.ts"; +import { Batch, type BatchError, Source } from "../../next/state/internal.ts"; +import type { Watchable } from "../../watch.ts"; +import type { Plugins, RolloutHost, RolloutStep } from "../plugins/plugins.ts"; +import type { PluginScript } from "../plugins/runtime.ts"; +import { type CrossedError, type CrossedReport, type FromWorker, uncrossed, uncrossedReport } from "./control.ts"; +import { type Exit, WorkerProcess } from "./process.ts"; + +/** What `createWorkers` takes. `kind`, `entry`, `execArgv` and `env` are bootstrap; the functions are called per use. */ +export type WorkersOptions

= { + /** The facet kind its workers load (`"worker"`), shipped by the plugins component. */ + readonly kind: string; + /** The worker's entry module (it calls `workerMain`), its Node flags and its environment. */ + readonly entry: string; + readonly execArgv?: readonly string[]; + readonly env?: Readonly>; + /** The route's `authorize`: whether `principal` may reach `address` (synchronous). */ + readonly authorize: (principal: P, address: Address) => boolean; + /** Who the worker at `address` is (stamped on its link). */ + readonly principal: (address: Address) => P; + /** The worker's start data (JSON), before each start; abortable (`close`), a throw fails the start. */ + readonly start: (address: Address, context: Context) => Promise | JsonValue; + /** The lock one live worker of `address` holds (across processes). */ + readonly lock: (address: Address) => string; + /** How long a worker idles before it is stopped: 5 minutes by default. */ + readonly idleMs?: () => number; + readonly plugins: Plugins; + /** A worker's output; without it, mirrored to this process's own. */ + readonly onOutput?: (address: Address, stream: "stdout" | "stderr", chunk: Uint8Array) => void; + readonly onError: (error: WorkersError) => void; +}; + +/** + * One address's worker (R5): running or on its way, or failed (until `retry` or a rollout) with a public error: a fixed + * message (`worker failed to start`, `worker exited (SIGSEGV)`) or the child's redacted `fatal`. + */ +export type WorkerState = + | { readonly status: "starting" | "running" | "stopping"; readonly pid: number; readonly startedAt: number } + | { readonly status: "failed"; readonly error: CrossedError; readonly at: number }; + +/** What the component reports (and the state's watch listeners that threw). */ +export type WorkersError = + | { + readonly kind: "worker_failed"; + readonly address: string; + /** The full error (a start's throw, a crash, the child's `fatal`): for the app's logs, never public. */ + readonly error: unknown; + readonly stderrTail?: string; + } + | { + /** The app's `authorize` threw: the hold is refused (`forbidden`), the error stays private. */ + readonly kind: "authorize_failed"; + readonly address: string; + readonly error: unknown; + } + | { + readonly kind: "worker_exited"; + readonly address: string; + readonly code: number | null; + readonly signal: string | null; + } + | BatchError; + +/** The workers component. */ +export interface Workers { + /** Per address key its `WorkerState`; no entry when stopped or never started. A JSON watchable. */ + readonly state: Watchable>>; + /** Suspends `address` while it has holders (`RouteHandle.suspend`), then stops its worker. */ + stop(address: Address, context: Context): Promise; + /** Clears `address`'s failure and suspension: it starts again now if held (`RouteHandle.retry`). */ + retry(address: Address, context: Context): Promise; + /** A rollout step shipping the kind's current set to every worker, one result per worker (§4). */ + step(name: string): RolloutStep; + /** Cancels starts, stops every worker; new starts reject `released/closed`. */ + close(context: Context): Promise; +} + +/** Creates the workers component on `host`: it routes `Kind` itself (`host.route`, before `load`). */ +export function createWorkers

( + host: Host

, + Kind: HostKind, + options: WorkersOptions>, + context: Context, +): Workers> { + // Its addresses are `Address>`, as the route hands them over. + const component = new WorkersComponent(options as unknown as WorkersOptions, context); + const route = host.route( + Kind, + { + authorize: (principal, address) => { + // A throw would reach the holder as the hold's failure: the app's error stays private. + try { + return options.authorize(principal, address); + } catch (error) { + options.onError({ kind: "authorize_failed", address: address.key, error }); + return false; + } + }, + // The principal is `options.principal`'s, a `P`. + resolve: (address, routing) => + component.resolve(address, routing) as Promise<{ link: Link; principal: P; trusted: true }>, + }, + context, + ); + component.route = route as unknown as RouteHandle; + return component as unknown as Workers>; +} + +/** + * A failure an address keeps until `retry` or a rollout: what the public sees (a fixed message, or the child's redacted + * `fatal`; decision "Slice 10 fix round, worker errors in public") and when. + */ +type Failure = { readonly error: CrossedError; readonly at: number }; + +/** + * Why a child fails when it exits, recorded as it happens: a start that failed, a `fatal`, a lost link. `error` is the + * full error (`onError` only); `public` what holders and `state` get (a fixed one from the exit when absent). + */ +type Cause = { + readonly error: unknown; + readonly public?: CrossedError; + /** The component ended the child itself (a failed start): its exit is no `worker_exited`. */ + readonly killed?: true; +}; + +/** How a child's reloads end once it is gone: stopped cleanly (a stop's exit) or with an error. */ +type Ended = { readonly kind: "stopped" } | { readonly kind: "failed"; readonly error: unknown }; + +/** One worker process of an address, with what its events recorded. */ +type Child = { + readonly process: WorkerProcess; + status: "starting" | "running" | "stopping"; + /** A stop was asked for: its exit is no crash. */ + stopping: boolean; + cause: Cause | undefined; + /** `started`: the app's host listens on its link. */ + readonly started: PromiseWithResolvers; + /** `ready`: its load report; rejected when it exits before. */ + readonly ready: PromiseWithResolvers; + /** Since when it is idle (from `ready`); `undefined` while busy or before `ready`. */ + idleSince: number | undefined; + /** An idle stop is queued (checked again when it runs). */ + idleStop: boolean; + /** The set its start message or its last reload that applied without failures carried. */ + shipped: string; + /** Its reloads, one outstanding at a time; settles to nothing (a report kept as its value would stay alive). */ + reloading: Promise; + /** The waiter of its outstanding reload. */ + reloaded: ((reply: Extract) => void) | undefined; + /** Set once its link is gone or it exited: every reload queued behind settles with it. */ + ended: Ended | undefined; + /** The grace timer of a stop or of a lost link. */ + grace: ReturnType | undefined; +}; + +/** One address: its operations in a queue, its child, its failure. Kept while it has any of them. */ +type Routed = { + readonly address: Address; + queue: Promise; + pending: number; + child: Child | undefined; + failure: Failure | undefined; + /** A start was queued and has not settled. */ + starting: Promise | undefined; +}; + +/** Starts a worker may take until `started` (fork, lock, connect): 60 s. */ +const START_TIMEOUT = 60_000; + +/** How long a stopping worker (or one whose link is gone) gets before its tree is killed: 5 s. */ +const GRACE = 5_000; + +/** How often idle workers are looked for: every second. */ +const IDLE_CHECK = 1_000; + +/** How long a worker idles before it is stopped unless `idleMs` says otherwise: 5 minutes. */ +const IDLE_MS = 5 * 60_000; + +/** The public error of a start that failed. */ +const START_FAILED: CrossedError = { message: "worker failed to start", code: "unavailable" }; + +/** The link type a route's `resolve` returns. */ +type Link = WorkerProcess["link"]; + +class WorkersComponent implements Workers { + /** Set by `createWorkers` once the route exists. */ + route!: RouteHandle; + readonly state: Watchable>>; + readonly #state: WorkersState; + readonly #options: WorkersOptions; + readonly #addresses = new Map(); + /** Aborted at `close`: starts waiting in the app's `start` or for `started` end. */ + readonly #closing = new AbortController(); + /** `#closing`'s signal as a context: what a start's waits are bounded by. */ + readonly #closingContext = withAbortSignal(this.#closing.signal, BACKGROUND_CONTEXT); + /** Whether the route's context closes it already (its first `resolve` attached that). */ + #routing = false; + #idleTimer: ReturnType | undefined; + + constructor(options: WorkersOptions, context: Context) { + this.#options = options; + this.#state = new WorkersState(this.#addresses, new Batch((error) => options.onError(error), undefined)); + // Its value is a `Record`. + this.state = this.#state as unknown as Watchable>>; + this.#closeWith(context); + } + + /** + * The route's `resolve`: a failed address rejects at once with its (public) failure, nothing forks; otherwise a + * start, after the previous child's exit. + */ + resolve(address: Address, routing: Context): Promise<{ link: Link; principal: Principal; trusted: true }> { + if (!this.#routing) { + this.#routing = true; + this.#closeWith(routing); + } + const routed = this.#routed(address); + if (routed.failure !== undefined) return Promise.reject(uncrossed(routed.failure.error)); + if (this.#closing.signal.aborted) return Promise.reject(closed()); + const started = this.#enqueue(routed, () => this.#start(routed, routing)); + routed.starting = started; + void started.then( + () => this.#settled(routed, started), + () => this.#settled(routed, started), + ); + return started; + } + + async stop(address: Address, context: Context): Promise { + this.route.suspend(address); + const routed = this.#addresses.get(address.key); + if (routed === undefined) return; + await awaitWithContext( + this.#enqueue(routed, () => this.#stop(routed)), + context, + ); + } + + async retry(address: Address, _context: Context): Promise { + const routed = this.#addresses.get(address.key); + if (routed !== undefined) { + routed.failure = undefined; + this.#state.refresh(); + this.#forget(routed); + } + this.route.retry(address); + } + + step(name: string): RolloutStep { + return { name, kinds: [this.#options.kind], reload: (_built, context) => this.#rollout(context) }; + } + + async close(context: Context): Promise { + if (!this.#closing.signal.aborted) this.#closing.abort(closed()); + clearInterval(this.#idleTimer); + this.#idleTimer = undefined; + const stops = [...this.#addresses.values()].map((routed) => this.#enqueue(routed, () => this.#stop(routed))); + await awaitWithContext(Promise.all(stops), context); + } + + /** `context`'s abort closes the component (the route's is aborted when the host closes). */ + #closeWith(context: Context): void { + const signal = context.abortSignal; + if (signal === undefined || signal.aborted) return; + signal.addEventListener("abort", () => void this.close(BACKGROUND_CONTEXT).catch(() => {}), { once: true }); + } + + /** The record of `address`, made by its first use. */ + #routed(address: Address): Routed { + let routed = this.#addresses.get(address.key); + if (routed === undefined) { + routed = { + address, + queue: Promise.resolve(), + pending: 0, + child: undefined, + failure: undefined, + starting: undefined, + }; + this.#addresses.set(address.key, routed); + } + return routed; + } + + /** Runs `operation` after the address's earlier ones. */ + #enqueue(routed: Routed, operation: () => Promise): Promise { + routed.pending++; + const run = routed.queue.then(operation); + routed.queue = run.then( + () => this.#done(routed), + () => this.#done(routed), + ); + return run; + } + + /** An operation ended. */ + #done(routed: Routed): void { + routed.pending--; + this.#forget(routed); + } + + /** A start settled: no longer starting. */ + #settled(routed: Routed, started: Promise): void { + if (routed.starting === started) routed.starting = undefined; + } + + /** Forgets an address with no operation, no child and no failure. */ + #forget(routed: Routed): void { + if (routed.pending === 0 && routed.child === undefined && routed.failure === undefined) { + this.#addresses.delete(routed.address.key); + } + } + + /** + * A start: the app's start data, the fork, the start message with the kind's current set, then `started` within the + * start timer (fork, lock and connect only). Its failure fails the address; `close` ends it at any of its waits (an + * app's `start` that ignores its signal too), a forked child is stopped. + */ + async #start(routed: Routed, routing: Context): Promise<{ link: Link; principal: Principal; trusted: true }> { + const { address } = routed; + // After the previous child's exit (its link may close first): a crash recorded meanwhile rejects this start. + await routed.child?.process.exited; + const crashed = routed.failure; + if (crashed !== undefined) throw uncrossed(crashed.error); + if (this.#closing.signal.aborted) throw closed(); + const context = withAbortSignal(this.#closing.signal, routing); + let data: JsonValue; + try { + data = await awaitWithContext(Promise.resolve(this.#options.start(address, context)), this.#closingContext); + } catch (error) { + if (this.#closing.signal.aborted) throw closed(); + throw this.#fail(routed, { error, public: START_FAILED }, undefined); + } + if (this.#closing.signal.aborted) throw closed(); + let child: Child; + try { + child = this.#fork(routed); + } catch (error) { + throw this.#fail(routed, { error, public: START_FAILED }, undefined); + } + const timer = setTimeout(() => { + const error = new Error(`worker ${address.key}: did not start within ${START_TIMEOUT / 1000} s`); + child.cause ??= { error, killed: true }; + child.process.kill(); + }, START_TIMEOUT); + // Every app hook of the start runs in here: what one throws stays private (`worker_failed`), its holders and + // `state` get the fixed public message (Astra's recheck R3: `principal` threw through to a guest). + try { + const scripts = await this.#options.plugins.scripts(this.#options.kind, context); + child.shipped = keyOf(scripts); + const lock = this.#options.lock(address); + child.process.send({ kind: "start", address, start: data, scripts, lock }); + await awaitWithContext(child.started.promise, this.#closingContext); + return { link: child.process.link, principal: this.#options.principal(address), trusted: true }; + } catch (error) { + // Closing is a stop (no failure); anything else fails the start. + if (this.#closing.signal.aborted) child.stopping = true; + else child.cause ??= { error, public: START_FAILED, killed: true }; + child.process.kill(); + await child.process.exited; + if (this.#closing.signal.aborted) throw closed(); + throw uncrossed(routed.failure?.error ?? START_FAILED); + } finally { + clearTimeout(timer); + } + } + + /** Forks the address's worker and records its events on it as they happen. */ + #fork(routed: Routed): Child { + const { address } = routed; + const onOutput = this.#options.onOutput; + let child!: Child; + const process = new WorkerProcess( + { + entry: this.#options.entry, + ...(this.#options.execArgv === undefined ? {} : { execArgv: this.#options.execArgv }), + ...(this.#options.env === undefined ? {} : { env: this.#options.env }), + ...(onOutput === undefined ? {} : { onOutput: (stream, chunk) => onOutput(address, stream, chunk) }), + }, + { + control: (message) => this.#control(routed, child, message), + linkLost: () => this.#linkLost(child), + linkClosed: () => this.#linkClosed(child), + exited: (exit) => this.#exited(routed, child, exit), + }, + ); + child = { + process, + status: "starting", + stopping: false, + cause: undefined, + started: Promise.withResolvers(), + ready: Promise.withResolvers(), + idleSince: undefined, + idleStop: false, + shipped: "", + reloading: Promise.resolve(), + reloaded: undefined, + ended: undefined, + grace: undefined, + }; + // Nobody may wait for them. + child.started.promise.catch(() => {}); + child.ready.promise.catch(() => {}); + routed.child = child; + this.#state.refresh(); + this.#watchIdle(); + return child; + } + + /** A control message of `child`. */ + #control(routed: Routed, child: Child, message: FromWorker): void { + switch (message.kind) { + case "started": + child.started.resolve(); + return; + case "ready": + child.status = "running"; + child.idleSince = message.idle ? Date.now() : undefined; + child.ready.resolve(message.report); + if (routed.child === child) this.#state.refresh(); + return; + case "idle": + child.idleSince = message.idle ? (child.idleSince ?? Date.now()) : undefined; + return; + case "fatal": + // Redacted by the child's `redact`: public as it is. + child.cause ??= { error: uncrossed(message.error), public: message.error }; + child.started.reject(uncrossed(message.error)); + child.process.kill(); + return; + case "reloaded": + child.reloaded?.(message); + } + } + + /** This side closed `child`'s link (the heartbeat, a protocol error): its child is stopped, its exit a crash. */ + #linkLost(child: Child): void { + this.#ended(child); + if (child.stopping) return; + child.cause ??= { error: new Error("worker: link lost") }; + this.#shutdown(child); + this.#state.refresh(); + } + + /** + * The child's side closed its link (it is exiting, or it closed its IPC and lives on): its reloads end; one that does + * not exit within the grace is killed, its exit a crash (link lost). + */ + #linkClosed(child: Child): void { + this.#ended(child); + if (child.stopping || child.grace !== undefined) return; + child.grace = setTimeout(() => { + child.cause ??= { error: new Error("worker: link lost") }; + child.process.kill(); + }, GRACE); + } + + /** + * `child` exited: a stop's exit forgets it; any other is a crash that fails the address (`worker_exited` and + * `worker_failed` once, the full error and stderr tail for the app's logs only), except once the component closes. + */ + #exited(routed: Routed, child: Child, exit: Exit): void { + clearTimeout(child.grace); + const clean = child.stopping && child.cause === undefined; + const cause: Cause = child.cause ?? { error: exitError(routed.address, exit, child) }; + const error = cause.public ?? publicExit(exit, child); + child.started.reject(uncrossed(error)); + child.ready.reject(uncrossed(error)); + this.#ended(child); + if (routed.child !== child) return; + routed.child = undefined; + if (!clean && !this.#closing.signal.aborted) { + const { code, signal } = exit; + // A failed start's kill is the component's own act (Opus's second recheck): only `worker_failed`. + if (cause.killed !== true) { + this.#options.onError({ kind: "worker_exited", address: routed.address.key, code, signal }); + } + this.#fail(routed, { ...cause, public: error }, child.process.stderrTail); + } + this.#state.refresh(); + this.#forget(routed); + } + + /** + * `child`'s link is gone or it exited (the first of these): its outstanding reload and every one queued behind it + * settle, `stopped` when a stop asked for it, else failed. + */ + #ended(child: Child): void { + if (child.ended !== undefined) return; + child.ended = + child.stopping && child.cause === undefined + ? { kind: "stopped" } + : { kind: "failed", error: new ChordError({ code: "unavailable", message: "worker: its link is gone" }) }; + child.reloaded?.({ kind: "reloaded", error: { message: "worker: its link is gone" } }); + } + + /** + * Fails `routed` with `cause` (`worker_failed` once, with the full error and `stderrTail`); returns the public error + * a start rejects with. + */ + #fail(routed: Routed, cause: Cause & { readonly public: CrossedError }, stderrTail: string | undefined): Error { + routed.failure = { error: cause.public, at: Date.now() }; + this.#options.onError({ + kind: "worker_failed", + address: routed.address.key, + error: cause.error, + ...(stderrTail === undefined ? {} : { stderrTail }), + }); + this.#state.refresh(); + return uncrossed(cause.public); + } + + /** A stop: `shutdown`, the grace, then the tree killed; resolves at its exit. */ + async #stop(routed: Routed): Promise { + const child = routed.child; + if (child === undefined) return; + child.stopping = true; + this.#shutdown(child); + this.#state.refresh(); + await child.process.exited; + } + + /** Asks `child` to shut down and kills its tree after the grace. */ + #shutdown(child: Child): void { + child.status = "stopping"; + child.process.send({ kind: "shutdown" }); + clearTimeout(child.grace); + child.grace = setTimeout(() => child.process.kill(), GRACE); + } + + /** Starts the idle check (every second) while any worker exists. */ + #watchIdle(): void { + if (this.#idleTimer !== undefined) return; + this.#idleTimer = setInterval(() => this.#checkIdle(), IDLE_CHECK); + this.#idleTimer.unref(); + } + + /** Queues a stop for every worker idle since `idleMs()`; stops checking once no worker is left. */ + #checkIdle(): void { + const idleMs = this.#idleMs(); + const now = Date.now(); + let workers = 0; + for (const routed of this.#addresses.values()) { + const child = routed.child; + if (child === undefined) continue; + workers++; + if (child.idleStop || child.status !== "running") continue; + if (child.idleSince === undefined || now - child.idleSince < idleMs) continue; + child.idleStop = true; + void this.#enqueue(routed, () => this.#idleStop(routed, child)); + } + if (workers === 0) { + clearInterval(this.#idleTimer); + this.#idleTimer = undefined; + } + } + + /** An idle stop when it runs: only if `child` is still the address's and still idle (a hold may have come since). */ + async #idleStop(routed: Routed, child: Child): Promise { + child.idleStop = false; + if (routed.child !== child || child.idleSince === undefined) return; + if (Date.now() - child.idleSince < this.#idleMs()) return; + await this.#stop(routed); + } + + /** How long a worker idles before it is stopped. */ + #idleMs(): number { + return this.#options.idleMs?.() ?? IDLE_MS; + } + + /** + * The rollout step (§4): the set as this rollout applied it, read now (before another rollout changes it); the + * failures recorded before it cleared and the held ones retried (listed once ready or failed); the set shipped to + * every worker whose last applied set differs, in parallel. `context`'s abort ends only this wait: the entries + * still waiting are listed with its error. + */ + async #rollout(context: Context): Promise<{ readonly hosts: readonly RolloutHost[] }> { + const { kind, plugins } = this.#options; + const scripts = plugins.current(kind) ?? (await plugins.scripts(kind, context)); + const key = keyOf(scripts); + const entries: { readonly host: string; readonly entry: Promise }[] = []; + for (const routed of [...this.#addresses.values()]) { + if (routed.failure === undefined) continue; + routed.failure = undefined; + this.#state.refresh(); + this.route.retry(routed.address); + const started = routed.starting; + if (started === undefined) this.#forget(routed); + else entries.push({ host: routed.address.key, entry: this.#retried(routed, started) }); + } + for (const routed of this.#addresses.values()) { + const child = routed.child; + if (child === undefined || child.status === "stopping" || child.shipped === key) continue; + entries.push({ host: routed.address.key, entry: this.#reload(routed.address.key, child, scripts, key) }); + } + return { hosts: await Promise.all(entries.map(({ host, entry }) => bounded(host, entry, context))) }; + } + + /** A retried address's entry: its start's load report once ready, or its error. */ + async #retried(routed: Routed, started: Promise): Promise { + const host = routed.address.key; + try { + await started; + const child = routed.child; + if (child === undefined) throw new Error("worker: exited"); + return { host, report: uncrossedReport(await child.ready.promise) }; + } catch (error) { + return { host, error }; + } + } + + /** + * Ships `scripts` to `child` after its outstanding reload; settles on `reloaded`, its link's loss or its exit. Only a + * reload that applied without failures marks the set shipped (a failed plugin is shipped again next time). + */ + #reload(host: string, child: Child, scripts: readonly PluginScript[], key: string): Promise { + const reload = child.reloading.then( + () => + new Promise((resolve) => { + const settle = (ended: Ended): void => { + resolve(ended.kind === "stopped" ? { host, stopped: true } : { host, error: ended.error }); + }; + if (child.ended !== undefined) { + settle(child.ended); + return; + } + child.reloaded = (reply) => { + child.reloaded = undefined; + if ("report" in reply) { + if (reply.report.failed.length === 0) child.shipped = key; + resolve({ host, report: uncrossedReport(reply.report) }); + } else if (child.ended !== undefined) { + settle(child.ended); + } else { + resolve({ host, error: uncrossed(reply.error) }); + } + }; + child.process.send({ kind: "reload", scripts }); + }), + ); + child.reloading = reload.then( + () => {}, + () => {}, + ); + return reload; + } +} + +/** What new starts reject with once the component closes. */ +function closed(): ChordError { + return new ChordError({ code: "released", reason: "closed", message: "workers: closed" }); +} + +/** The full error of an exit nobody asked for (the app's logs). */ +function exitError(address: Address, { code, signal }: Exit, child: Child): Error { + const how = signal === null ? `code ${code}` : `signal ${signal}`; + const when = child.status === "starting" ? "exited before it was ready" : "exited"; + return new ChordError({ code: "unavailable", message: `worker ${address.key}: ${when} (${how})` }); +} + +/** The public error of an exit: a start that failed, or the exit's code or signal. */ +function publicExit({ code, signal }: Exit, child: Child): CrossedError { + if (child.status === "starting") return START_FAILED; + return { message: `worker exited (${signal ?? `code ${code}`})`, code: "unavailable" }; +} + +/** A set's identity: its plugins and digests. */ +function keyOf(scripts: readonly PluginScript[]): string { + return scripts + .map(({ plugin, built }) => `${plugin.name}@${built.kind === "script" ? built.digest : "failed"}`) + .sort() + .join("\n"); +} + +/** `entry`, or `host` listed with the abort's error once `context` aborts first (the entry goes on). */ +async function bounded(host: string, entry: Promise, context: Context): Promise { + try { + return await awaitWithContext(entry, context); + } catch (error) { + return { host, error }; + } +} + +/** The workers' state: a JSON watchable of every address's `WorkerState`, published once per microtask turn. */ +class WorkersState extends Source { + readonly #addresses: ReadonlyMap; + #value: Readonly> | undefined; + #scheduled = false; + + constructor(addresses: ReadonlyMap, batch: Batch) { + super(batch); + this.#addresses = addresses; + } + + get value(): JsonValue { + this.#value ??= stateOf(this.#addresses); + // Plain JSON: statuses, numbers and crossed errors. + return this.#value as unknown as JsonValue; + } + + deliver(context: Context): void { + const value = this.value; + this.publish(value, [["r", value]], false, context); + } + + /** Something changed: published with the turn's other changes while someone watches or derives from it. */ + refresh(): void { + this.#value = undefined; + if (this.#scheduled || (!this.watched && this.dependents.size === 0)) return; + this.#scheduled = true; + queueMicrotask(() => { + this.#scheduled = false; + this.batch.within(() => this.changed(), BACKGROUND_CONTEXT); + }); + } +} + +/** Every address's state: its child's, or its failure; none for an address with neither. */ +function stateOf(addresses: ReadonlyMap): Record { + const state: Record = {}; + for (const [key, { child, failure }] of addresses) { + if (failure !== undefined) { + state[key] = { status: "failed", ...failure }; + } else if (child !== undefined) { + const { pid, startedAt } = child.process; + state[key] = { status: child.status, pid, startedAt }; + } + } + return state; +} diff --git a/packages/chord/src/next-react/README.md b/packages/chord/src/next-react/README.md new file mode 100644 index 000000000..11ee4c3af --- /dev/null +++ b/packages/chord/src/next-react/README.md @@ -0,0 +1,14 @@ +# react + +React bindings (slice 11 §3, decision "Slice 11, P7"): `@earendil-works/chord/next/react` once v2 has its entry point; +optional (React is an optional peer), non-React apps never load it. No components, no styling. + +- `HostProvider({ host })`, `useHost()`. +- `useHold(Type, params)`: a reference held while the component is mounted, released on unmount, held again when its + params change (compared by value); `undefined` on the first render. +- `useWatch(watchable)`: its current value (`undefined` while it has none), re-rendering on change + (`useSyncExternalStore` over `watchStore`). +- `useRegistry(R)`: the registry's composed value; `useRefState(ref)`: the reference's state. +- A component unmounted by a reload has not left (design §21): its releases are plain releases. + +Tests: `test/next-node/react/` (`stores.test.ts` in Node, `hooks.test.ts` in a real browser). diff --git a/packages/chord/src/next-react/hooks.ts b/packages/chord/src/next-react/hooks.ts new file mode 100644 index 000000000..a3731a920 --- /dev/null +++ b/packages/chord/src/next-react/hooks.ts @@ -0,0 +1,79 @@ +// React bindings (slice 11 §3, decision "Slice 11, P7"; p2's `useObject`, `useReadable`, `useRegistry`): a host in +// context, references held for a component's life, and watchables read with `useSyncExternalStore`. A component +// unmounted by a reload has not left (design §21): its releases are plain releases, presence is the app's. No +// components, no styling. + +import { + createContext, + createElement, + type ReactNode, + useContext, + useEffect, + useMemo, + useState, + useSyncExternalStore, +} from "react"; +import { BACKGROUND_CONTEXT, withAbortSignal } from "../context/index.ts"; +import type { Contract, ParamsOf } from "../next/contracts/index.ts"; +import type { Principal } from "../next/data/index.ts"; +import { canonicalJson } from "../next/data/principal.ts"; +import type { Held, Host, RefState } from "../next/host/index.ts"; +import type { Registry } from "../next/registries/index.ts"; +import type { NativeWatchable, Watchable } from "../watch.ts"; +import { watchStore } from "./stores.ts"; + +/** The host components use, given by `HostProvider`. */ +const HostContext = createContext | undefined>(undefined); + +/** Gives `host` to the components below. */ +export function HostProvider

(props: { readonly host: Host

; readonly children?: ReactNode }) { + // Components hold for the app's principal type: the context keeps it as the host was given. + return createElement(HostContext.Provider, { value: props.host as unknown as Host }, props.children); +} + +/** The host `HostProvider` gives; throws outside one. */ +export function useHost

(): Host

{ + const host = useContext(HostContext); + if (host === undefined) throw new Error("useHost: no HostProvider above this component"); + return host as unknown as Host

; +} + +/** + * A reference to `Type` with `params`, held while the component is mounted and released when it unmounts or its params + * change (compared by value); `undefined` before its first hold (the first render). + */ +export function useHold(Type: C, params: ParamsOf): Held | undefined { + const host = useHost(); + const key = canonicalJson(params as never, BACKGROUND_CONTEXT); + const [held, setHeld] = useState<{ readonly key: string; readonly ref: Held } | undefined>(undefined); + // biome-ignore lint/correctness/useExhaustiveDependencies: `key` is `params` by value: an inline object must not re-hold. + useEffect(() => { + const ref = host.hold(Type, params, BACKGROUND_CONTEXT); + setHeld({ key, ref }); + return () => ref.release(); + }, [host, Type, key]); + return held !== undefined && held.key === key ? held.ref : undefined; +} + +/** The current value of `watchable` (a state member, a derive), re-rendering on change; `undefined` while it has none. */ +export function useWatch(watchable: Watchable | NativeWatchable | undefined): T | undefined { + const store = useMemo(() => watchStore(watchable), [watchable]); + return useSyncExternalStore(store.subscribe, store.snapshot, store.snapshot); +} + +/** Registry `R`'s composed value on the component's host, re-rendering on change; `undefined` until it is usable. */ +export function useRegistry(R: Registry): V | undefined { + const host = useHost(); + const [value, setValue] = useState | undefined>(undefined); + useEffect(() => { + const abort = new AbortController(); + host.registry(R, withAbortSignal(abort.signal, BACKGROUND_CONTEXT)).then(setValue, () => {}); + return () => abort.abort(); + }, [host, R]); + return useWatch(value); +} + +/** The state of `ref` (loading, live, failed, ended), re-rendering on change; `undefined` without a reference. */ +export function useRefState(ref: { readonly state: NativeWatchable } | undefined): RefState | undefined { + return useWatch(ref?.state); +} diff --git a/packages/chord/src/next-react/index.ts b/packages/chord/src/next-react/index.ts new file mode 100644 index 000000000..03a78df79 --- /dev/null +++ b/packages/chord/src/next-react/index.ts @@ -0,0 +1,5 @@ +// Chord v2's React bindings (`@earendil-works/chord/next/react` once v2 has its entry point; slice 11 §3): optional, +// non-React apps never load them. + +export { HostProvider, useHold, useHost, useRefState, useRegistry, useWatch } from "./hooks.ts"; +export { type ExternalStore, watchStore } from "./stores.ts"; diff --git a/packages/chord/src/next-react/stores.ts b/packages/chord/src/next-react/stores.ts new file mode 100644 index 000000000..4b9e47e09 --- /dev/null +++ b/packages/chord/src/next-react/stores.ts @@ -0,0 +1,45 @@ +// External stores over Chord's watchables (slice 11 §3): what React's `useSyncExternalStore` reads. A store's snapshot +// is the watchable's current value (`undefined` while it has none: a reference not bound yet, ended, or a member its +// variant lacks; nothing throws during a render); subscribing starts a watch whose every call notifies React, and the +// unsubscribe ends it. + +import { BACKGROUND_CONTEXT, withAbortSignal } from "../context/index.ts"; +import type { NativeWatchable, Watchable } from "../watch.ts"; + +/** What `useSyncExternalStore` takes: subscribe (returning its unsubscribe) and the current snapshot. */ +export type ExternalStore = { + readonly subscribe: (changed: () => void) => () => void; + readonly snapshot: () => T | undefined; +}; + +/** The store of `source` (a state member, a derive, a registry value, `ref.state`); none for `undefined`. */ +export function watchStore(source: Watchable | NativeWatchable | undefined): ExternalStore { + return { + subscribe: (changed) => { + if (source === undefined) return () => {}; + const abort = new AbortController(); + const context = withAbortSignal(abort.signal, BACKGROUND_CONTEXT); + // A watch that cannot start (its source ended, the abort came first) only means no more changes. + source.watch(context).then( + (watch) => { + (watch.start as (listener: () => Promise) => void)(async () => { + changed(); + }); + // A watch starts at the value it acquired, which may be newer than the snapshot React read (a + // reference's members bind after the first render): its calls bring only later changes. + changed(); + }, + () => {}, + ); + return () => abort.abort(); + }, + snapshot: () => { + if (source === undefined) return undefined; + try { + return source.value; + } catch { + return undefined; + } + }, + }; +} diff --git a/packages/chord/src/next-tui/README.md b/packages/chord/src/next-tui/README.md new file mode 100644 index 000000000..31a5f5f8f --- /dev/null +++ b/packages/chord/src/next-tui/README.md @@ -0,0 +1,29 @@ +# tui (sandbox side) + +The terminal client's code inside the sandbox (slice 11 §4, decisions "Slice 11, P8" and "P9"): +`@earendil-works/chord/next/tui` once v2 has its entry point. The launcher is `next-node/tui`, the server side +`next-node/surfaces` (`tuiSurface`). + +```ts +// the app's terminal entry (its base bundle) +startTerminalRuntime

({ + modules: { "@acme/app/v3": api, "@acme/app/v3/tui": apiTui }, + core: [{ name: "@acme/tui", facet: tuiCore }], + render: ({ host, terminal, theme, copy, open, attach, exit }) => { + const scope = new TuiScope(host, () => redraw()); // hold, watch, registry; dispose + terminal.onInput((data) => …); terminal.write(frame); + theme.fg("accent", "Notes"); // colours by role + }, +}); +``` + +- `bridge.ts`: the messages between the sandbox and the launcher (input, resizes, relayed link sockets, requests with + replies: a plugin script, the server's base, `copy`, `open`; `outdated`, `exit`, `failed`). The bootstrap leaves the + bridge on `globalThis.chordBridge`; the runtime takes it off before any plugin runs. +- `channel.ts`: the sandbox's side: requests, `RelayLink` (a Chord link whose bytes the launcher relays), input and + resizes. `terminal.ts`: the bridged terminal (size, write, input, resize). `theme.ts`: the theme lookup (`fg`, `bg`, + `onChange`). `runtime.ts`: the client (`startClient` with `TuiScripts`; scripts run with `vm`; no wakes). +- `scope.ts`: `TuiScope(host, changed)`: `hold`, `watch` (a reader, `undefined` while no value), `registry`, `dispose` + (no `changed` after it). References wait and rebind by themselves; no terminal library dependency. + +Tests: `test/next/tui/` and, with the launcher and sandbox, `test/next-node/tui/`. diff --git a/packages/chord/src/next-tui/bridge.ts b/packages/chord/src/next-tui/bridge.ts new file mode 100644 index 000000000..8de935383 --- /dev/null +++ b/packages/chord/src/next-tui/bridge.ts @@ -0,0 +1,69 @@ +// The terminal's bridge (slice 11 §4, decision "Slice 11, P8"): the only way between the sandbox, where the app's +// terminal code and the plugins run, and the trusted launcher outside it. The launcher sends input, resizes and the +// relayed bytes of the Chord link; the sandbox sends frames for the terminal, opens and feeds link sockets through the +// launcher, asks for plugin scripts and the server's base hash, and asks for actions on the machine (copy, open a URL, +// attach a file the user names), which the launcher confirms with the user. Messages are plain data, framed over the +// sandbox's pipes (`next-node/tui/pipe.ts`); requests carry an id their reply has. + +/** What the launcher tells the sandbox at its start. */ +export type BridgeStart = { + /** The base bundle's hash (the sandbox runs it). */ + readonly base: string; + /** The text of the combined plugin scripts at start. */ + readonly combined: string; + readonly columns: number; + readonly rows: number; + /** How many colours the terminal shows. */ + readonly colors: "truecolor" | "256"; + /** The terminal's background: which of the user's two themes applies. */ + readonly mode: "dark" | "light"; +}; + +/** A request of the sandbox the launcher answers. */ +export type BridgeRequest = + | { readonly kind: "fetch"; readonly path: string } + | { readonly kind: "base" } + | { readonly kind: "copy"; readonly text: string } + | { readonly kind: "open"; readonly url: string } + /** A file of the machine the user names (`path`): its name and bytes once confirmed, else `null`. */ + | { readonly kind: "attach"; readonly path: string }; + +/** What the sandbox sends. */ +export type SandboxMessage = + | { readonly kind: "write"; readonly data: string } + | { readonly kind: "socket_open"; readonly socket: number } + | { readonly kind: "socket_send"; readonly socket: number; readonly bytes: Uint8Array } + | { readonly kind: "socket_close"; readonly socket: number } + | { readonly kind: "request"; readonly id: number; readonly request: BridgeRequest } + /** The sandbox's code is outdated (a new base bundle): the launcher starts a new one. */ + | { readonly kind: "outdated" } + /** The base bundle failed to start (its message; the launcher shows none of the sandbox's own output). */ + | { readonly kind: "failed"; readonly message: string } + | { readonly kind: "exit"; readonly code: number }; + +/** What the launcher sends. */ +export type LauncherMessage = + | { readonly kind: "input"; readonly data: string } + | { readonly kind: "resize"; readonly columns: number; readonly rows: number } + | { readonly kind: "socket_opened"; readonly socket: number } + | { readonly kind: "socket_message"; readonly socket: number; readonly bytes: Uint8Array } + | { readonly kind: "socket_closed"; readonly socket: number } + | { readonly kind: "reply"; readonly id: number; readonly value: unknown } + /** An attach request's answer: the file the user confirmed. */ + | { readonly kind: "attached"; readonly id: number; readonly name: string; readonly bytes: Uint8Array } + | { readonly kind: "refused"; readonly id: number; readonly message: string }; + +/** The bridge as the sandbox's code gets it from the bootstrap (once: it is taken off `globalThis`). */ +export type Bridge = { + readonly start: BridgeStart; + send(message: SandboxMessage): void; + onMessage(listener: (message: LauncherMessage) => void): void; + /** + * Runs `source` (registered plugin scripts) in the sandbox's context, as the bootstrap ran the base bundle: with no + * dynamic import callback (`import()` throws). + */ + evaluate(source: string, filename: string): void; +}; + +/** The global the sandbox's bootstrap leaves the bridge on, for the base bundle's runtime to take. */ +export const BRIDGE = "chordBridge"; diff --git a/packages/chord/src/next-tui/channel.ts b/packages/chord/src/next-tui/channel.ts new file mode 100644 index 000000000..d7ed7755c --- /dev/null +++ b/packages/chord/src/next-tui/channel.ts @@ -0,0 +1,163 @@ +// The sandbox's side of the bridge (slice 11 §4): one dispatcher of the launcher's messages (input, resizes, link +// sockets, replies), requests with their replies, and Chord links whose bytes the launcher relays to the server. + +import { LinkEnd } from "../next/link/link.ts"; +import type { Bridge, BridgeRequest, BridgeStart, LauncherMessage } from "./bridge.ts"; +import { splitFrames } from "./frames.ts"; + +/** What one relayed socket reports: it opened, a frame arrived. */ +export type RelayEvents = { opened(): void; received(): void }; + +/** The sandbox's bridge: requests, relayed sockets, and the terminal's input and resizes. */ +export class SandboxChannel { + readonly #bridge: Bridge; + readonly #replies = new Map(); + readonly #sockets = new Map(); + readonly #input = new Set<(data: string) => void>(); + readonly #resize = new Set<(columns: number, rows: number) => void>(); + #next = 0; + + constructor(bridge: Bridge) { + this.#bridge = bridge; + bridge.onMessage((message) => this.#dispatch(message)); + } + + /** What the launcher told at the start. */ + get start(): BridgeStart { + return this.#bridge.start; + } + + /** Asks the launcher; rejects with its refusal. */ + request(request: BridgeRequest): Promise { + const id = ++this.#next; + return new Promise((resolve, reject) => { + this.#replies.set(id, { resolve, reject }); + this.#bridge.send({ kind: "request", id, request }); + }); + } + + /** Runs the registered scripts `text` (named `filename` in stacks) through the sandbox's own evaluation. */ + evaluate(text: string, filename: string): void { + this.#bridge.evaluate(text, filename); + } + + /** Writes `data` to the terminal, in frames the launcher takes (`splitFrames`). */ + write(data: string): void { + for (const frame of splitFrames(data)) this.#bridge.send({ kind: "write", data: frame }); + } + + /** Tells the launcher this sandbox's code is outdated. */ + outdated(): void { + this.#bridge.send({ kind: "outdated" }); + } + + /** Ends the terminal with `code`. */ + exit(code: number): void { + this.#bridge.send({ kind: "exit", code }); + } + + /** Calls `listener` with each input (keys, pastes) until the returned removal. */ + onInput(listener: (data: string) => void): () => void { + this.#input.add(listener); + return () => this.#input.delete(listener); + } + + /** Calls `listener` at each resize until the returned removal. */ + onResize(listener: (columns: number, rows: number) => void): () => void { + this.#resize.add(listener); + return () => this.#resize.delete(listener); + } + + /** Opens a socket to the server's links endpoint through the launcher: its link, reporting to `events`. */ + openSocket(events: RelayEvents): RelayLink { + const socket = ++this.#next; + const link = new RelayLink(socket, events, this.#bridge, () => this.#sockets.delete(socket)); + this.#sockets.set(socket, link); + this.#bridge.send({ kind: "socket_open", socket }); + return link; + } + + /** One message of the launcher, to whom it is for. */ + #dispatch(message: LauncherMessage): void { + switch (message.kind) { + case "input": + for (const listener of this.#input) listener(message.data); + return; + case "resize": + for (const listener of this.#resize) listener(message.columns, message.rows); + return; + case "socket_opened": + this.#sockets.get(message.socket)?.opened(); + return; + case "socket_message": + this.#sockets.get(message.socket)?.received(message.bytes); + return; + case "socket_closed": + this.#sockets.get(message.socket)?.dropped(); + return; + case "reply": + case "attached": + case "refused": + this.#settle(message); + } + } + + /** Settles the request a reply or refusal answers. */ + #settle(message: Extract): void { + const waiting = this.#replies.get(message.id); + this.#replies.delete(message.id); + if (message.kind === "reply") waiting?.resolve(message.value); + else if (message.kind === "attached") waiting?.resolve({ name: message.name, bytes: message.bytes }); + else waiting?.reject(new Error(message.message)); + } +} + +/** A Chord link over a socket the launcher holds: bytes relayed both ways, buffered until the socket opened. */ +export class RelayLink extends LinkEnd { + readonly #socket: number; + readonly #events: RelayEvents; + readonly #bridge: Bridge; + readonly #forget: () => void; + /** Messages sent while the socket connects; `undefined` once open. */ + #connecting: Uint8Array[] | undefined = []; + + constructor(socket: number, events: RelayEvents, bridge: Bridge, forget: () => void) { + super(); + this.#socket = socket; + this.#events = events; + this.#bridge = bridge; + this.#forget = forget; + } + + send(message: Uint8Array): void { + if (this.closed) return; + if (this.#connecting !== undefined) this.#connecting.push(message); + else this.#bridge.send({ kind: "socket_send", socket: this.#socket, bytes: message }); + } + + close(): void { + if (this.closed) return; + this.#bridge.send({ kind: "socket_close", socket: this.#socket }); + this.dropped(); + } + + /** The launcher's socket opened: what was sent meanwhile goes out. */ + opened(): void { + const queued = this.#connecting ?? []; + this.#connecting = undefined; + this.#events.opened(); + for (const message of queued) this.send(message); + } + + /** A frame arrived. */ + received(bytes: Uint8Array): void { + this.#events.received(); + this.deliver(bytes); + } + + /** The socket closed (either side): the link ends. */ + dropped(): void { + this.#forget(); + this.ended(); + } +} diff --git a/packages/chord/src/next-tui/frames.ts b/packages/chord/src/next-tui/frames.ts new file mode 100644 index 000000000..29f3d5d91 --- /dev/null +++ b/packages/chord/src/next-tui/frames.ts @@ -0,0 +1,51 @@ +// Frames for the terminal (slice 11 §4; fix round 1, Fable L9): the launcher drops a frame over 64 KiB whole (an escape +// it could hide is never written in parts), so the sandbox splits a large write into frames, each cut only between +// escape sequences and never inside a surrogate pair: a first paint of a large true-colour screen arrives whole. + +/** The most UTF-16 units of one frame: at most 3 UTF-8 bytes each, so under the launcher's 64 KiB. */ +export const FRAME_UNITS = 16 * 1024; + +/** `data` as frames of at most `FRAME_UNITS` units, cut outside escape sequences (one longer is its own frame). */ +export function splitFrames(data: string): string[] { + if (data.length <= FRAME_UNITS) return [data]; + const frames: string[] = []; + let start = 0; + // The last place a frame may end: after a character or a whole sequence, never inside one. + let safe = 0; + let index = 0; + while (index < data.length) { + const end = data.charCodeAt(index) === 27 ? sequenceEnd(data, index) : unitEnd(data, index); + if (end - start > FRAME_UNITS && safe > start) { + frames.push(data.slice(start, safe)); + start = safe; + } + index = end; + safe = end; + } + frames.push(data.slice(start)); + return frames; +} + +/** The index after the character at `index`, a surrogate pair kept together. */ +function unitEnd(data: string, index: number): number { + const code = data.charCodeAt(index); + return code >= 0xd800 && code <= 0xdbff && index + 1 < data.length ? index + 2 : index + 1; +} + +/** The index after the escape sequence at `index` (CSI to its final; OSC, DCS and the like to their terminator). */ +function sequenceEnd(data: string, index: number): number { + const introducer = data[index + 1]; + if (introducer === "[") { + let at = index + 2; + while (at < data.length && data.charCodeAt(at) >= 0x20 && data.charCodeAt(at) <= 0x3f) at++; + return Math.min(at + 1, data.length); + } + if (introducer !== undefined && "]PX^_".includes(introducer)) { + for (let at = index + 2; at < data.length; at++) { + if (data.charCodeAt(at) === 7) return at + 1; + if (data[at] === "\x1b" && data[at + 1] === "\\") return at + 2; + } + return data.length; + } + return Math.min(index + 2, data.length); +} diff --git a/packages/chord/src/next-tui/index.ts b/packages/chord/src/next-tui/index.ts new file mode 100644 index 000000000..fd5db3903 --- /dev/null +++ b/packages/chord/src/next-tui/index.ts @@ -0,0 +1,14 @@ +// Chord v2's terminal surface, the sandbox side (`@earendil-works/chord/next/tui` once v2 has its entry point; slice +// 11 §4): `startTerminalRuntime` in the app's terminal entry, the bridged terminal, the theme lookup and `TuiScope`. +// The launcher and the server side are `next/tui/launcher` and `next/web/server`'s `tuiSurface`. + +export type { Bridge, BridgeRequest, BridgeStart, LauncherMessage, SandboxMessage } from "./bridge.ts"; +export { + type AttachedFile, + startTerminalRuntime, + type TerminalRuntime, + type TerminalRuntimeOptions, +} from "./runtime.ts"; +export { TuiScope } from "./scope.ts"; +export { BridgeTerminal } from "./terminal.ts"; +export { TerminalTheme } from "./theme.ts"; diff --git a/packages/chord/src/next-tui/runtime.ts b/packages/chord/src/next-tui/runtime.ts new file mode 100644 index 000000000..d89e34f7c --- /dev/null +++ b/packages/chord/src/next-tui/runtime.ts @@ -0,0 +1,155 @@ +// `startTerminalRuntime` (slice 11 §4, decisions "Slice 11, P8" and "P9"): the terminal client inside the sandbox, in +// the app's terminal entry (the base bundle the launcher downloaded). A client host like a page's: the plugin scripts +// the launcher brought, loaded with the app's core facets and the kit's themes; the link to the server relayed by the +// launcher, kept by the same connection loop; `TuiScripts` followed to apply new plugin scripts in place; a new base +// bundle hands over to the launcher, which starts the new one. The app gets the host, the bridged terminal, the theme +// lookup and the launcher's actions. + +import { BACKGROUND_CONTEXT } from "../context/index.ts"; +import type { Principal } from "../next/data/index.ts"; +import type { PluginFacet } from "../next/host/index.ts"; +import { + type Client, + type ClientError, + type ClientModules, + type ConnectionState, + FALLBACK_THEMES, + type FacetFunction, + followActiveTheme, + REGISTER, + type RegisteredScript, + startClient, + TuiScripts, + themesFacet, +} from "../next/surfaces/index.ts"; +import { BRIDGE, type Bridge } from "./bridge.ts"; +import { SandboxChannel } from "./channel.ts"; +import { BridgeTerminal } from "./terminal.ts"; +import { TerminalTheme } from "./theme.ts"; + +/** What `startTerminalRuntime` takes. */ +export type TerminalRuntimeOptions

= { + /** The modules the terminal plugins import, by specifier: the app's API modules for the kind, its components. */ + readonly modules: ClientModules; + /** The app's own terminal facets. */ + readonly core?: readonly PluginFacet

[]; + readonly canRetry?: () => Promise; + readonly onConnection?: (state: ConnectionState) => void; + /** Draws the app, once its plugins are loaded. */ + readonly render: (runtime: TerminalRuntime

) => void; + /** Plugins left out or kept old, a failed update, the host's errors; dropped by default (stderr is the launcher's). */ + readonly onError?: (error: ClientError) => void; +}; + +/** A file the launcher read for the sandbox: its name (the path's last segment) and its bytes. */ +export type AttachedFile = { readonly name: string; readonly bytes: Uint8Array }; + +/** What the app's terminal code gets. */ +export type TerminalRuntime

= { + readonly host: Client

["host"]; + readonly terminal: BridgeTerminal; + readonly theme: TerminalTheme; + /** Asks the launcher to copy `text` to the machine's clipboard; it asks the user. Whether it was done. */ + copy(text: string): Promise; + /** Asks the launcher to open `url` (http or https) in a browser; it asks the user. Whether it was done. */ + open(url: string): Promise; + /** + * Asks the launcher for the file at `path` on the machine (at most 16 MiB); it asks the user. Its name and bytes, or + * `undefined` when declined. + */ + attach(path: string): Promise; + /** Ends the terminal client with `code`. */ + exit(code: number): void; +}; + +/** Starts the terminal client inside the sandbox; resolves with its runtime once rendered. */ +export async function startTerminalRuntime

( + options: TerminalRuntimeOptions

, +): Promise> { + const channel = new SandboxChannel(takeBridge()); + const { start } = channel; + const scripts = new SandboxScripts(channel); + const core: PluginFacet

[] = [{ name: "chord.themes", facet: themesFacet

() }, ...(options.core ?? [])]; + const client = await startClient

({ + scripts: TuiScripts, + base: start.base, + initial: scripts.run(start.combined), + modules: options.modules, + core, + open: (events) => channel.openSocket(events), + fetchScript: async (entry) => { + const text = (await channel.request({ kind: "fetch", path: entry.url })) as string; + const registered = scripts + .run(text) + .find(({ plugin, digest }) => plugin === entry.name && digest === entry.digest); + if (registered === undefined) throw new Error(`plugin "${entry.name}": ${entry.url} did not register`); + return registered; + }, + serverBase: () => + channel.request({ kind: "base" }).then( + (base) => base as string, + () => undefined, + ), + outdated: () => channel.outdated(), + ...(options.canRetry === undefined ? {} : { canRetry: options.canRetry }), + ...(options.onConnection === undefined ? {} : { onConnection: options.onConnection }), + environment: TERMINAL_ENVIRONMENT, + onError: options.onError ?? (() => {}), + }); + const theme = new TerminalTheme(FALLBACK_THEMES[start.mode], start.colors); + await followActiveTheme(client.host, start.mode, (active) => theme.set(active), BACKGROUND_CONTEXT); + const runtime: TerminalRuntime

= { + host: client.host, + terminal: new BridgeTerminal(channel), + theme, + copy: (text) => channel.request({ kind: "copy", text }).then(Boolean), + open: (url) => channel.request({ kind: "open", url }).then(Boolean), + // The launcher's reply: its own data, `null` when declined. + attach: (path) => + channel.request({ kind: "attach", path }).then((file) => (file as AttachedFile | null) ?? undefined), + exit: (code) => channel.exit(code), + }; + options.render(runtime); + return runtime; +} + +/** The bridge the bootstrap left on `globalThis`, taken off it (plugin code never finds it there). */ +function takeBridge(): Bridge { + const scope = globalThis as Record; + const bridge = scope[BRIDGE] as Bridge | undefined; + if (bridge === undefined) throw new Error("startTerminalRuntime: not inside the terminal's sandbox (no bridge)"); + delete scope[BRIDGE]; + return bridge; +} + +/** A terminal has no page wakes: the loop retries on its backoff alone. */ +const TERMINAL_ENVIRONMENT = { + now: () => Date.now(), + after: (ms: number, callback: () => void) => { + const timer = setTimeout(callback, ms); + return () => clearTimeout(timer); + }, + online: () => true, + onWake: () => () => {}, +}; + +/** Runs registered scripts' text in the sandbox and collects what registered (`chordFacet`). */ +class SandboxScripts { + readonly #channel: SandboxChannel; + /** The scripts the text being run registers. */ + #collected: RegisteredScript[] = []; + + constructor(channel: SandboxChannel) { + this.#channel = channel; + (globalThis as Record)[REGISTER] = (plugin: string, digest: string, run: FacetFunction) => { + this.#collected.push({ plugin, digest, run }); + }; + } + + /** Runs `text` (one or more registered scripts): what it registered. */ + run(text: string): RegisteredScript[] { + this.#collected = []; + this.#channel.evaluate(text, "chord-plugins.js"); + return this.#collected; + } +} diff --git a/packages/chord/src/next-tui/scope.ts b/packages/chord/src/next-tui/scope.ts new file mode 100644 index 000000000..8302e4d02 --- /dev/null +++ b/packages/chord/src/next-tui/scope.ts @@ -0,0 +1,81 @@ +// `TuiScope` (slice 11 §4, decision "Slice 11, P9"; p2's `TuiScope`): a terminal component's Chord resources, the +// counterpart of the React bindings for terminal UI libraries. It holds references, follows watchables and registries, +// calls one `changed` callback (the app's redraw) on every change, and releases everything at `dispose`. References +// wait and rebind by themselves (no retry loop, unlike p2's); no terminal library dependency. + +import { BACKGROUND_CONTEXT, withAbortSignal } from "../context/index.ts"; +import type { Contract, ParamsOf } from "../next/contracts/index.ts"; +import type { Principal } from "../next/data/index.ts"; +import type { Held, Host } from "../next/host/index.ts"; +import type { Registry } from "../next/registries/index.ts"; +import type { NativeWatchable, Watchable } from "../watch.ts"; + +/** A component's scope: what it holds and follows, until `dispose`. */ +export class TuiScope

{ + readonly #host: Host

; + readonly #changed: () => void; + readonly #abort = new AbortController(); + readonly #releases: (() => void)[] = []; + + constructor(host: Host

, changed: () => void) { + this.#host = host; + this.#changed = changed; + } + + /** A reference to `Type` with `params`, held until `dispose`; its state changes call `changed`. */ + hold(Type: C, params: ParamsOf): Held { + const held = this.#host.hold(Type, params, this.#context()); + this.#releases.push(() => held.release()); + this.watch(held.state); + return held; + } + + /** + * Follows `source` (a member, a derive, `ref.state`) until `dispose`: its changes call `changed`. Returns a reader of + * its current value (`undefined` while it has none: nothing throws in a redraw). + */ + watch(source: Watchable | NativeWatchable): () => T | undefined { + source.watch(this.#context()).then( + (watch) => { + (watch.start as (listener: () => Promise) => void)(async () => { + if (!this.#abort.signal.aborted) this.#changed(); + }); + // The value it acquired may be newer than what was drawn (a reference bound since): draw it. + if (!this.#abort.signal.aborted) this.#changed(); + }, + // Disposed (or the source ended) before the watch started: nothing to follow. + () => {}, + ); + return () => { + try { + return source.value; + } catch { + return undefined; + } + }; + } + + /** Registry `R`'s composed value, followed until `dispose`; a reader, `undefined` until it is usable. */ + registry(R: Registry): () => V | undefined { + let read: () => V | undefined = () => undefined; + this.#host.registry(R, this.#context()).then( + (value) => { + read = this.watch(value); + if (!this.#abort.signal.aborted) this.#changed(); + }, + () => {}, + ); + return () => read(); + } + + /** Releases every reference and ends every watch; `changed` is not called after it. */ + dispose(): void { + this.#abort.abort(); + for (const release of this.#releases.splice(0)) release(); + } + + /** The context of the scope's work: aborted at `dispose`. */ + #context() { + return withAbortSignal(this.#abort.signal, BACKGROUND_CONTEXT); + } +} diff --git a/packages/chord/src/next-tui/terminal.ts b/packages/chord/src/next-tui/terminal.ts new file mode 100644 index 000000000..a29723cf4 --- /dev/null +++ b/packages/chord/src/next-tui/terminal.ts @@ -0,0 +1,45 @@ +// The terminal as the app's terminal code sees it inside the sandbox (slice 11 §4): its size, writing frames, and +// input and resizes, all through the bridge; the launcher holds the real terminal. Library-agnostic: an app adapts it +// to its terminal UI library (pi-tui's `Terminal` in a few lines). + +import type { SandboxChannel } from "./channel.ts"; + +/** The bridged terminal. */ +export class BridgeTerminal { + readonly #channel: SandboxChannel; + #columns: number; + #rows: number; + + constructor(channel: SandboxChannel) { + this.#channel = channel; + this.#columns = channel.start.columns; + this.#rows = channel.start.rows; + channel.onResize((columns, rows) => { + this.#columns = columns; + this.#rows = rows; + }); + } + + get columns(): number { + return this.#columns; + } + + get rows(): number { + return this.#rows; + } + + /** Writes `data` (text and escape sequences) to the terminal. */ + write(data: string): void { + this.#channel.write(data); + } + + /** Calls `listener` with each input (keys, pastes, as the terminal sends them) until the returned removal. */ + onInput(listener: (data: string) => void): () => void { + return this.#channel.onInput(listener); + } + + /** Calls `listener` after each resize (the new size read from `columns` and `rows`) until the returned removal. */ + onResize(listener: () => void): () => void { + return this.#channel.onResize(() => listener()); + } +} diff --git a/packages/chord/src/next-tui/theme.ts b/packages/chord/src/next-tui/theme.ts new file mode 100644 index 000000000..724356e3d --- /dev/null +++ b/packages/chord/src/next-tui/theme.ts @@ -0,0 +1,44 @@ +// The terminal's theme lookup (slice 11 §4b, decision "Looks and themes"): colours by role from the active theme, as +// the terminal shows them (true colour, else the nearest of 256; a theme's `tui` overrides first). A plugin asks for a +// role (`theme.fg("accent", "Notes")`), never writes raw colours; a new theme applies at the next redraw. + +import { type ColorDepth, type Theme, terminalColor } from "../next/surfaces/index.ts"; + +/** Colours by role for the terminal. */ +export class TerminalTheme { + readonly #depth: ColorDepth; + #theme: Theme; + readonly #listeners = new Set<() => void>(); + + constructor(theme: Theme, depth: ColorDepth) { + this.#theme = theme; + this.#depth = depth; + } + + /** The active theme. */ + get theme(): Theme { + return this.#theme; + } + + /** `text` in `role`'s colour (the default colour after it). */ + fg(role: string, text: string): string { + return `\x1b[${terminalColor(this.#theme, role, this.#depth, "fg")}m${text}\x1b[39m`; + } + + /** `text` on `role`'s colour as its background. */ + bg(role: string, text: string): string { + return `\x1b[${terminalColor(this.#theme, role, this.#depth, "bg")}m${text}\x1b[49m`; + } + + /** Calls `listener` when the active theme changes (to redraw) until the returned removal. */ + onChange(listener: () => void): () => void { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + /** The active theme is now `theme`. */ + set(theme: Theme): void { + this.#theme = theme; + for (const listener of this.#listeners) listener(); + } +} diff --git a/packages/chord/src/next-web/README.md b/packages/chord/src/next-web/README.md new file mode 100644 index 000000000..0c946b782 --- /dev/null +++ b/packages/chord/src/next-web/README.md @@ -0,0 +1,25 @@ +# web (browser side) + +`startPage` (slice 11 §2) for the app's web entry, the base bundle the server builds: `@earendil-works/chord/next/web` +once v2 has its entry point. The server side is `next-node/surfaces` (`webSurface`). + +```ts +startPage

({ + connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), + modules: { "@acme/app/v3": api, "@acme/app/v3/web": apiWeb, react: React, "react/jsx-runtime": jsx }, + core: [{ name: "@acme/web", facet: webCore }], + canRetry: async () => true, // false: stop reconnecting (a guest's ended share) + onConnection: (state) => setBanner(state), // connecting | connected | waiting + render: (host) => createRoot(root).render(), +}); +``` + +- The page's inline bootstrap (`PageAssets.bootstrap`, the response's nonce) defines `chordFacet` before the combined + file runs; its scripts queue, and `startPage` takes them (`scripts.ts`). Later scripts load with `` }; + }, + }, + }, + framers: ["https://shell.example.com"], + csp: { "img-src": ["https://cdn.example.com"] }, + }), + }); + const page = await send(server.port, "GET", "/page"); + expect(page.headers).toMatchObject({ + "cache-control": "no-store", + "x-content-type-options": "nosniff", + "referrer-policy": "no-referrer", + }); + expect(page.headers["x-frame-options"]).toBeUndefined(); + expect(page.headers["content-security-policy"]).toBe( + `default-src 'none'; script-src 'self' 'nonce-${nonce}'; connect-src 'self' ws://127.0.0.1:${server.port}; base-uri 'none'; form-action 'self'; frame-ancestors https://shell.example.com; img-src https://cdn.example.com`, + ); + // Fable's recheck: a nonce read from one page must not run a script injected into another. + const first = nonce; + await send(server.port, "GET", "/page"); + expect(nonce).not.toBe(first); + const missing = await send(server.port, "GET", "/nope"); + expect([missing.status, missing.headers["x-content-type-options"]]).toEqual([404, "nosniff"]); + expect(missing.headers["content-security-policy"]).toMatch(/'nonce-[A-Za-z0-9+/]{22}=='/); + expect(missing.headers["content-security-policy"]).not.toContain(nonce); + await server.close(); + const plain = await served({}); + expect((await send(plain.port, "GET", "/")).headers["x-frame-options"]).toBe("DENY"); + await plain.close(); + }); + + it("lets a relay's verified request through with its framer", async () => { + const server = await served({ + "@app/who": whoami, + "@app/relay": contributing({ + relays: [ + { + name: "radius", + verify: (request) => + request.headers.get("x-relay") === "secret" + ? { origin: "https://me.relay.example", framer: "https://relay.example" } + : undefined, + }, + ], + }), + }); + const forwarded = { "x-forwarded-for": "203.0.113.9" }; + expect((await send(server.port, "GET", "/whoami", { headers: forwarded })).status).toBe(403); + const relayed = await send(server.port, "GET", "/whoami", { headers: { ...forwarded, "x-relay": "secret" } }); + expect(relayed.status).toBe(200); + expect(relayed.headers["content-security-policy"]).toContain("frame-ancestors https://relay.example"); + await server.close(); + }); +}); + +describe("routes and wraps failing", () => { + it("answers a throwing route 500 without its message, a throwing wrap 500 naming its plugin, an HttpFailure its status", async () => { + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /throws": { + public: true, + handler: async () => { + throw new Error("secret"); + }, + }, + "GET /failure": { + public: true, + handler: async () => { + throw new HttpFailure(422, "no"); + }, + }, + }, + }), + "@app/b": contributing({ + routes: { "GET /wrapped": { public: true, handler: async () => ({ kind: "json", value: 1 }) } }, + wraps: [ + { + name: "broken", + phase: "headers", + wrap: async () => { + throw new Error("wrap"); + }, + }, + ], + }), + }); + const thrown = await send(server.port, "GET", "/throws"); + expect([thrown.status, thrown.body.toString()]).toEqual([500, "Internal Server Error"]); + expect((await send(server.port, "GET", "/failure")).status).toBe(422); + expect((await send(server.port, "GET", "/wrapped")).status).toBe(500); + expect(server.errors).toMatchObject([ + { kind: "route_failed", pattern: "GET /throws", plugin: "@app/a", error: { message: "secret" } }, + { kind: "wrap_failed", name: "broken", plugin: "@app/b", error: { message: "wrap" } }, + ]); + await server.close(); + }); + + it("runs wraps outermost first around the route, in its request's scope", async () => { + const order: string[] = []; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /r": { + public: true, + handler: async (_request, ctx) => { + order.push(`route as ${ctx.principal?.kind ?? "anonymous"}`); + return { kind: "json", value: 1 }; + }, + }, + }, + wraps: [ + { + name: "inner", + phase: "transform", + wrap: async (request, next) => { + order.push("transform"); + return next(request); + }, + }, + { + name: "outer", + phase: "observe", + wrap: async (request, next) => { + order.push("observe"); + const response = await next(request); + return { ...response, headers: { "X-Observed": "1" } }; + }, + }, + ], + }), + }); + const answer = await send(server.port, "GET", "/r", { headers: { "x-principal": "owner" } }); + expect([order, answer.headers["x-observed"]]).toEqual([["observe", "transform", "route as owner"], "1"]); + await server.close(); + }); +}); + +describe("reloads and close", () => { + it("a reload swapping a route leaves a running request on the old router; the next uses the new one", async () => { + const gate = Promise.withResolvers(); + const old = contributing({ + routes: { + "GET /v": { + public: true, + handler: async () => { + await gate.promise; + return { kind: "html", body: "old" }; + }, + }, + }, + }); + const server = await served({ "@app/v": old }); + const running = send(server.port, "GET", "/v"); + await new Promise((resolve) => setTimeout(resolve, 50)); + const next = contributing({ + routes: { "GET /v": { public: true, handler: async () => ({ kind: "html", body: "new" }) } }, + }); + await server.host.reload(server.facets({ "@app/v": next }), context); + expect((await send(server.port, "GET", "/v")).body.toString()).toBe("new"); + gate.resolve(); + expect((await running).body.toString()).toBe("old"); + await server.close(); + }); + + it("close refuses new requests (503), waits for running ones up to closeTimeout, then seals them (503)", async () => { + let sealed: AbortSignal | undefined; + const server = await served( + { + "@app/slow": contributing({ + routes: { + "GET /quick": { + public: true, + handler: async () => { + await new Promise((resolve) => setTimeout(resolve, 30)); + return { kind: "html", body: "done" }; + }, + }, + "GET /stuck": { + public: true, + handler: async (_request, ctx: Ctx) => { + sealed = ctx.abortSignal; + return new Promise(() => {}); + }, + }, + }, + }), + }, + { config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 100 } }, + ); + const quick = send(server.port, "GET", "/quick"); + const stuck = send(server.port, "GET", "/stuck"); + await new Promise((resolve) => setTimeout(resolve, 10)); + const closing = server.http.close(context); + expect((await server.http.fetch(loopback("/quick"), LOOPBACK)).status).toBe(503); + expect((await quick).body.toString()).toBe("done"); + await closing; + expect(sealed?.aborted).toBe(true); + await server.host.close(context); + expect((await stuck).status).toBe(503); + await expect(send(server.port, "GET", "/quick")).rejects.toThrow(); + }); + + // Astra's build review: a request authenticating when close began ran its route afterwards. + it("never runs a route for a request still authenticating when close sealed it", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let calls = 0; + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /": { + public: true, + handler: async () => { + calls++; + return { kind: "json", value: 1 }; + }, + }, + }, + }), + }, + { + config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 20 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + gate.resolve(); + expect([(await answer).status, calls]).toEqual([503, 0]); + await server.close(); + }); +}); + +describe("mounting", () => { + it("without a connection a request is never local", async () => { + const server = await served({ "@app/who": whoami }, { config: { listen: undefined } }); + expect((await server.http.fetch(loopback("/whoami"))).status).toBe(403); + await server.close(); + }); +}); + +describe("cancellation", () => { + // Sol's second recheck C: after cancellation the chain still started the next authenticator. + it("starts no further authenticator once close sealed the request", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + let later = 0; + const server = await served( + { + "@app/r": contributing({ + routes: { "GET /": { public: true, handler: async () => ({ kind: "html", body: "ok" }) } }, + }), + "@app/z-later": defineFacet((_host, self) => + self.contribute(Authenticators, () => { + later++; + return { kind: "owner" }; + }), + ), + }, + { + config: { listen: undefined, closeTimeout: 10 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return undefined; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + expect((await answer).status).toBe(503); + gate.resolve(); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(later).toBe(0); + await server.close(); + }); + + it("asks no authenticator for a request aborted before it arrived", async () => { + let calls = 0; + const server = await served( + { + "@app/r": contributing({ + routes: { "GET /": { public: true, handler: async () => ({ kind: "html", body: "ok" }) } }, + }), + }, + { + config: { listen: undefined }, + authenticate: () => { + calls++; + return undefined; + }, + }, + ); + await server.http.fetch(loopback("/", { signal: AbortSignal.abort() }), LOOPBACK); + expect(calls).toBe(0); + await server.close(); + }); + + // Sol's third recheck: a request aborted while its route returned the `bytes` description got a stream born sealed, + // which never errored; the request never finished. + it.each(["inside the route", "after the route returned"] as const)( + "ends a request whose bytes Response was born sealed (aborted %s)", + async (when) => { + const abort = new AbortController(); + const entered = Promise.withResolvers(); + let reads = 0; + let bornSealed: boolean | undefined; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": { + public: true, + handler: async (_request, ctx) => { + entered.resolve(); + if (when === "inside the route") abort.abort(new Error("client gone")); + bornSealed = ctx.abortSignal?.aborted; + return { + kind: "bytes", + mime: "application/octet-stream", + read: async () => { + reads++; + return new Uint8Array([1]); + }, + }; + }, + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1_000 } }, + ); + const answer = server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + await entered.promise; + if (when === "after the route returned") abort.abort(new Error("client gone")); + await answer; + await new Promise((resolve) => setImmediate(resolve)); + const started = Date.now(); + await server.http.close(context); + expect([Date.now() - started < 500, reads, bornSealed]).toEqual([true, 0, when === "inside the route"]); + await server.close(); + }, + ); + + // Sol's second recheck B: an aborted `Response` nobody touched again kept its request: close waited its bound. + it("ends a request whose bytes Response was aborted and dropped without another read", async () => { + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": { + public: true, + handler: async () => ({ + kind: "bytes", + mime: "application/octet-stream", + read: async () => new Uint8Array([1]), + }), + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1_000 } }, + ); + const abort = new AbortController(); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + abort.abort(new Error("client gone")); + await new Promise((resolve) => setImmediate(resolve)); + const started = Date.now(); + await server.http.close(context); + expect(Date.now() - started).toBeLessThan(500); + await server.close(); + }); + + // Astra's recheck R4, Sol's N1: a close before the first resolution started a stopped watch; its throw ended the process. + it("a close before the publication leaves nothing running: the waiting request answers 503", async () => { + const server = await served({}, { load: false, config: { listen: undefined } }); + const waiting = server.http.fetch(loopback("/x"), LOOPBACK); + await server.http.close(withAbortSignal(AbortSignal.abort(), context)); + await server.load(); + expect((await waiting).status).toBe(503); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.close(); + }); + + it("a close while the first request resolves the registries starts no follow", async () => { + const server = await served({ "@app/who": whoami }, { config: { listen: undefined, closeTimeout: 10 } }); + const pending = server.http.fetch(loopback("/whoami"), LOOPBACK); + await server.http.close(withAbortSignal(AbortSignal.timeout(20), context)); + expect([200, 503]).toContain((await pending).status); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.close(); + }); + + // Sol's recheck N9. + it("a close whose context is aborted already does not wait", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const server = await served( + { + "@app/wait": contributing({ + routes: { + "GET /wait": { + public: true, + handler: async () => { + entered.resolve(); + await gate.promise; + return { kind: "html", body: "done" }; + }, + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 1000 } }, + ); + const answer = server.http.fetch(loopback("/wait"), LOOPBACK); + await entered.promise; + const started = Date.now(); + await server.http.close(withAbortSignal(AbortSignal.abort(), context)); + expect(Date.now() - started).toBeLessThan(500); + gate.resolve(); + await answer; + await server.close(); + }); + + // Sol's recheck N5: an authenticator ignoring its signal kept fetch pending after close returned. + it("close settles a request whose authenticator ignores its signal (503)", async () => { + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const server = await served( + { + "@app/r": contributing({ + routes: { "GET /": { public: true, handler: async () => ({ kind: "html", body: "ok" }) } }, + }), + }, + { + config: { listen: undefined, closeTimeout: 10 }, + authenticate: async () => { + entered.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const answer = server.http.fetch(loopback("/"), LOOPBACK); + await entered.promise; + await server.http.close(context); + const settled = await Promise.race([answer.then((response) => response.status), sleepThen(30, "pending")]); + expect(settled).toBe(503); + gate.resolve(); + await server.close(); + }); + + // Astra's recheck R2, Sol's N3: a held `Response` went on calling `read` after its scope was sealed. + it.each(["the request aborted", "the component closed"] as const)("reads no bytes after %s", async (kind) => { + let reads = 0; + let readAfterSeal = false; + const abort = new AbortController(); + const server = await served( + { + "@app/bytes": contributing({ + routes: { + "GET /stream": { + public: true, + handler: async () => ({ + kind: "bytes", + mime: "application/octet-stream", + read: async (_offset, _length, ctx) => { + reads++; + readAfterSeal ||= ctx.abortSignal?.aborted ?? false; + return new Uint8Array([reads]); + }, + }), + }, + }, + }), + }, + { config: { listen: undefined, closeTimeout: 10 } }, + ); + const response = await server.http.fetch(loopback("/stream", { signal: abort.signal }), LOOPBACK); + const reader = response.body!.getReader(); + await reader.read(); + if (kind === "the request aborted") abort.abort(new Error("client gone")); + else await server.http.close(context); + await new Promise((resolve) => setImmediate(resolve)); + await expect(reader.read()).rejects.toThrow(); + expect({ reads, readAfterSeal }).toEqual({ reads: 1, readAfterSeal: false }); + await server.close(); + }); + + // Sol's recheck N4. + it("a body that errors after valid JSON fails the request (400); the route's side effect never runs", async () => { + let writes = 0; + const server = await served( + { + "@app/body": contributing({ + routes: { + "POST /body": { + public: true, + handler: async (request) => { + const value = await request.json(); + writes++; + return { kind: "json", value }; + }, + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + let pulls = 0; + const body = new ReadableStream({ + pull(controller) { + if (pulls++ === 0) controller.enqueue(new TextEncoder().encode("{}")); + else controller.error(new Error("incoming body checksum failed")); + }, + }); + const init = { method: "POST", body, duplex: "half" as const }; + const response = await server.http.fetch(loopback("/body", init), LOOPBACK); + expect([response.status, writes]).toEqual([400, 0]); + await server.close(); + }); +}); + +describe("answers", () => { + // Sol's recheck N8: `Response` threw on a body for these statuses. + it.each([204, 205, 304])("answers %s without a body", async (status) => { + const server = await served( + { + "@app/s": contributing({ + routes: { "GET /": { public: true, handler: async () => ({ kind: "json", value: null, status }) } }, + }), + }, + { config: { listen: undefined } }, + ); + const response = await server.http.fetch(loopback("/"), LOOPBACK); + expect([response.status, response.body, response.headers.has("content-length")]).toEqual([status, null, false]); + await server.close(); + }); + + // Fable's recheck N3: the kernel's `forbidden` reached the client as a reported 500. + it("answers a forbidden escaping a route 403, unreported", async () => { + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /": { + public: true, + handler: async () => { + throw new ChordError({ code: "forbidden", message: "no" }); + }, + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + expect((await server.http.fetch(loopback("/"), LOOPBACK)).status).toBe(403); + expect(server.errors).toEqual([]); + await server.close(); + }); + + // Opus's recheck R2: one value per header name allowed one cookie per response. + it("sends several Set-Cookie headers", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /": { + public: true, + handler: async () => ({ kind: "json", value: 1, headers: { "Set-Cookie": ["a=1", "b=2"] } }), + }, + }, + }), + }); + expect((await send(server.port, "GET", "/")).headers["set-cookie"]).toEqual(["a=1", "b=2"]); + const fetched = await server.http.fetch( + new Request(`http://127.0.0.1:${server.port}/`, { headers: { host: `127.0.0.1:${server.port}` } }), + { localAddress: "127.0.0.1", localPort: server.port }, + ); + expect(fetched.headers.getSetCookie()).toEqual(["a=1", "b=2"]); + await server.close(); + }); +}); + +describe("GC", () => { + // Sol's second recheck B, her GC probe: 8 of 8 such requests kept their scopes. + it("collects the scopes of aborted bytes Responses dropped without another read", async () => { + const refs: WeakRef[] = []; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": { + public: true, + handler: async (_request, ctx) => { + refs.push(new WeakRef(ctx)); + return { + kind: "bytes", + mime: "application/octet-stream", + read: async () => new Uint8Array([1]), + }; + }, + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + for (let i = 0; i < 8; i++) { + const abort = new AbortController(); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + abort.abort(new Error("client gone")); + } + await expectCollected(...refs); + await server.close(); + }); + + it("collects the scopes of bytes Responses born sealed (aborted while the route returned)", async () => { + const refs: WeakRef[] = []; + const aborts: AbortController[] = []; + const server = await served( + { + "@app/b": contributing({ + routes: { + "GET /": { + public: true, + handler: async (_request, ctx) => { + refs.push(new WeakRef(ctx)); + aborts.at(-1)?.abort(new Error("client gone")); + return { + kind: "bytes", + mime: "application/octet-stream", + read: async () => new Uint8Array([1]), + }; + }, + }, + }, + }), + }, + { config: { listen: undefined } }, + ); + // In a function of its own: the test's frame keeps no last request. + const fetchAborted = async (): Promise => { + const abort = new AbortController(); + aborts.push(abort); + await server.http.fetch(loopback("/", { signal: abort.signal }), LOOPBACK); + }; + for (let i = 0; i < 8; i++) await fetchAborted(); + aborts.length = 0; + await expectCollected(...refs); + await server.close(); + }); + + it("collects requests' scopes and requests once answered", async () => { + const refs: WeakRef[] = []; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /r": { + public: true, + handler: async (request, ctx) => { + refs.push(new WeakRef(ctx), new WeakRef(request)); + return { kind: "json", value: 1 }; + }, + }, + }, + }), + }); + for (let i = 0; i < 5; i++) await send(server.port, "GET", "/r"); + await expectCollected(...refs); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/node.test.ts b/packages/chord/test/next-node/http/node.test.ts new file mode 100644 index 000000000..c8aaf0de6 --- /dev/null +++ b/packages/chord/test/next-node/http/node.test.ts @@ -0,0 +1,227 @@ +// Tests of Chord's own Node server (the listener's native): request targets (origin-form only; a malformed or +// double-slash path never crashes or reroutes), raw `Host` headers, Web requests made only when read, the request's scope lasting until its response is +// flushed (and `close` waiting for it), and a client's disconnect aborting the scope. + +import { once } from "node:events"; +import { createServer, request as httpRequest, type IncomingMessage } from "node:http"; +import { connect } from "node:net"; +import { describe, expect, it } from "vitest"; +import { nodeRequest } from "../../../src/next-node/http/index.ts"; +import { context, contributing, served } from "./support.ts"; + +/** What `port` answers a raw request `text` with: its status line and body. */ +async function raw(port: number, text: string): Promise<{ readonly status: string; readonly body: string }> { + const socket = connect(port, "127.0.0.1"); + let out = ""; + socket.on("data", (chunk) => { + out += chunk.toString("latin1"); + }); + socket.on("error", () => {}); + await once(socket, "connect"); + socket.write(text); + await once(socket, "close"); + const [head = "", body = ""] = out.split("\r\n\r\n"); + return { status: head.split("\r\n")[0] ?? "", body }; +} + +/** A route answering its path and params, under `GET /who` and `GET /n/:name`. */ +const echo = contributing({ + routes: { + "GET /who": { public: true, handler: async (request) => ({ kind: "json", value: request.path }) }, + "GET /n/:name": { public: true, handler: async (request) => ({ kind: "json", value: request.params }) }, + }, +}); + +describe("targets", () => { + // Astra's build review, Fable's security audit C1: `GET //[` threw out of the request handler and ended the process. + it("answers a malformed or double-slash path without crashing or rerouting it (Opus's audit H5)", async () => { + const server = await served({ "@app/echo": echo }); + const host = `Host: 127.0.0.1:${server.port}`; + const get = (target: string) => + raw(server.port, `GET ${target} HTTP/1.1\r\n${host}\r\nConnection: close\r\n\r\n`); + expect((await get("//[")).status).toBe("HTTP/1.1 404 Not Found"); + expect((await get("//evil.example/who")).status).toBe("HTTP/1.1 404 Not Found"); + expect(await get("/a/../who")).toEqual({ status: "HTTP/1.1 200 OK", body: '"/who"' }); + // The URL parser resolves `%2e%2e` as a dot segment: `/n/%2e%2e` is `/`. + expect((await get("/n/%2e%2e")).status).toBe("HTTP/1.1 404 Not Found"); + expect((await get("/n/a%5Cb")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get("/n/..%2F..%2Fsecret")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get("/who")).status).toBe("HTTP/1.1 200 OK"); + await server.close(); + }); + + it("refuses an absolute-form target, two Host headers and none (403)", async () => { + const server = await served({ "@app/echo": echo }); + const host = `Host: 127.0.0.1:${server.port}`; + const answers = await Promise.all([ + raw(server.port, `GET http://evil.example/who HTTP/1.1\r\n${host}\r\nConnection: close\r\n\r\n`), + raw(server.port, `GET /who HTTP/1.1\r\n${host}\r\nHost: evil.example\r\nConnection: close\r\n\r\n`), + raw(server.port, "GET /who HTTP/1.0\r\n\r\n"), + ]); + expect(answers.map(({ status }) => status)).toEqual([ + "HTTP/1.1 403 Forbidden", + "HTTP/1.1 403 Forbidden", + "HTTP/1.1 403 Forbidden", + ]); + await server.close(); + }); + + it("refuses a Host with userinfo (403) and a method a Web Request cannot carry (501)", async () => { + const server = await served({ "@app/echo": echo }); + const answers = await Promise.all([ + raw(server.port, `GET /who HTTP/1.1\r\nHost: attacker@127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`), + raw(server.port, `TRACE /who HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nConnection: close\r\n\r\n`), + ]); + // `TRACK` never reaches Chord: Node's parser refuses it (400); `CONNECT` goes to Node's `connect` event. + expect(answers.map(({ status }) => status)).toEqual(["HTTP/1.1 403 Forbidden", "HTTP/1.1 501 Not Implemented"]); + await server.close(); + }); +}); + +describe("mounting on an app's own Node server", () => { + // Sol's recheck N6: a hand-made `Request` threw unhandled on `Host: [` or an absolute-form target (the process + // ended) and kept only the first of two `Host`s. + it("converts with the listener's checks, never throwing: bad targets, userinfo, TRACE refused; duplicate Hosts kept", async () => { + const server = await served({ "@app/echo": echo }, { config: { listen: undefined } }); + const own = createServer(async (message, res) => { + const converted = nodeRequest(message, res); + if (converted.kind === "refused") return void res.writeHead(converted.status).end(); + const response = await server.http.fetch(converted.request, converted.connection); + for (const [name, value] of response.headers) res.appendHeader(name, value); + res.writeHead(response.status).end(Buffer.from(await response.arrayBuffer())); + }); + await new Promise((resolve) => own.listen(0, "127.0.0.1", resolve)); + const address = own.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + const host = `Host: 127.0.0.1:${port}`; + const get = (text: string) => raw(port, `${text}Connection: close\r\n\r\n`); + expect(await get(`GET /who HTTP/1.1\r\n${host}\r\n`)).toEqual({ status: "HTTP/1.1 200 OK", body: '"/who"' }); + expect((await get("GET /who HTTP/1.1\r\nHost: [\r\n")).status).toBe("HTTP/1.1 400 Bad Request"); + expect((await get(`GET http://evil.example/who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 403 Forbidden"); + expect((await get(`GET /who HTTP/1.1\r\n${host}\r\nHost: evil.example\r\n`)).status).toBe( + "HTTP/1.1 403 Forbidden", + ); + // Sol's second recheck A: both threw out of the conversion; the app's callback ended the process. + expect((await get(`TRACE /who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 501 Not Implemented"); + expect((await get(`GET /who HTTP/1.1\r\nHost: attacker@127.0.0.1:${port}\r\n`)).status).toBe( + "HTTP/1.1 403 Forbidden", + ); + expect((await get(`GET /who HTTP/1.1\r\n${host}\r\n`)).status).toBe("HTTP/1.1 200 OK"); + await new Promise((resolve) => own.close(resolve)); + await server.close(); + }); +}); + +describe("Web requests", () => { + it("makes a Web Request only when something reads the headers or the body (here: no authenticator does)", async () => { + const server = await served( + { + "@app/r": contributing({ + routes: { + "GET /plain": { public: true, handler: async (request) => ({ kind: "json", value: request.path }) }, + "GET /headers": { + public: true, + handler: async (request) => ({ kind: "json", value: request.headers.get("x-thing") }), + }, + "POST /body": { + public: true, + handler: async (request) => ({ kind: "json", value: await request.json() }), + }, + }, + }), + }, + { authenticate: () => undefined }, + ); + const original = globalThis.Request; + let made = 0; + globalThis.Request = class extends original { + constructor(...args: ConstructorParameters) { + super(...args); + made++; + } + }; + try { + const host = `Host: 127.0.0.1:${server.port}\r\nConnection: close`; + await raw(server.port, `GET /plain HTTP/1.1\r\n${host}\r\n\r\n`); + expect(made).toBe(0); + const read = await raw(server.port, `GET /headers HTTP/1.1\r\n${host}\r\nX-Thing: t\r\n\r\n`); + const posted = await raw(server.port, `POST /body HTTP/1.1\r\n${host}\r\nContent-Length: 3\r\n\r\n"a"`); + expect([read.body, posted.body, made]).toEqual(['"t"', '"a"', 2]); + } finally { + globalThis.Request = original; + } + await server.close(); + }); +}); + +describe("the request's scope", () => { + // Sol's build review: the scope sealed, and `close` stopped waiting, before a large response was flushed. + it("lasts until the response is flushed; close waits for it", async () => { + const size = 32 * 1024 * 1024; + let signal: AbortSignal | undefined; + const server = await served( + { + "@app/large": contributing({ + routes: { + "GET /large": { + public: true, + handler: async (_request, ctx) => { + signal = ctx.abortSignal; + return { kind: "html", body: "x".repeat(size) }; + }, + }, + }, + }), + }, + { config: { listen: { host: "127.0.0.1", port: 0 }, closeTimeout: 2000 } }, + ); + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/large", agent: false }); + const response = await new Promise((resolve) => { + request.on("response", (arrived: IncomingMessage) => { + arrived.pause(); + resolve(arrived); + }); + request.end(); + }); + let length = 0; + response.on("data", (chunk: Buffer) => { + length += chunk.length; + }); + response.pause(); + await new Promise((resolve) => setTimeout(resolve, 20)); + const sealedBeforeFlush = signal?.aborted; + const closing = server.http.close(context); + await new Promise((resolve) => setTimeout(resolve, 20)); + response.resume(); + await once(response, "end"); + await closing; + expect({ sealedBeforeFlush, length }).toEqual({ sealedBeforeFlush: false, length: size }); + await server.close(); + }); + + it("is aborted when the client disconnects", async () => { + const entered = Promise.withResolvers(); + const server = await served({ + "@app/slow": contributing({ + routes: { + "GET /slow": { + public: true, + handler: async (_request, ctx) => { + entered.resolve(ctx.abortSignal!); + return new Promise(() => {}); + }, + }, + }, + }), + }); + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/slow", agent: false }); + request.on("error", () => {}); + request.end(); + const signal = await entered.promise; + const aborted = new Promise((resolve) => signal.addEventListener("abort", resolve, { once: true })); + request.destroy(); + await aborted; + expect(signal.aborted).toBe(true); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/request.test.ts b/packages/chord/test/next-node/http/request.test.ts new file mode 100644 index 000000000..9bbfb491a --- /dev/null +++ b/packages/chord/test/next-node/http/request.test.ts @@ -0,0 +1,130 @@ +// Tests of requests (slice 10 §2, D4): the summary a route gets, `json` and `body` with their limits and statuses +// (400, 413, an `HttpFailure` escaping a route answering its status), and a body nobody read (drained or closed). + +import { Agent, request as httpRequest } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { HttpRequest } from "../../../src/next-node/http/index.ts"; +import { contributing, send, served } from "./support.ts"; + +describe("the request a route gets", () => { + it("has its method, path, params, query, headers, origin, provenance and the response's nonce", async () => { + let seen: HttpRequest | undefined; + const server = await served({ + "@app/a": contributing({ + routes: { + "GET /notes/:id": { + public: true, + handler: async (request) => { + seen = request; + return { kind: "json", value: null }; + }, + }, + }, + }), + }); + await send(server.port, "GET", "/notes/a%20b?x=1&x=2", { headers: { "X-Thing": "t" } }); + expect(seen).toMatchObject({ + method: "GET", + path: "/notes/a%20b", + params: { id: "a b" }, + origin: `http://127.0.0.1:${server.port}`, + provenance: { kind: "local" }, + }); + expect([seen?.query.getAll("x"), seen?.headers.get("x-thing")]).toEqual([["1", "2"], "t"]); + expect(seen?.nonce).toMatch(/^[A-Za-z0-9+/]{22}==$/); + await server.close(); + }); +}); + +describe("bodies", () => { + /** A server whose `POST /json` answers the JSON it read (limit 16), `POST /body` the bytes it read (limit 16). */ + const reading = () => + served({ + "@app/a": contributing({ + routes: { + "POST /json": { + public: true, + handler: async (request) => ({ kind: "json", value: await request.json(16) }), + }, + "POST /body": { + public: true, + handler: async (request) => { + let bytes = 0; + for await (const chunk of request.body(16)) bytes += chunk.byteLength; + return { kind: "json", value: bytes }; + }, + }, + }, + }), + }); + + it("parses JSON within its limit; malformed is 400, larger is 413 (declared or streamed)", async () => { + const server = await reading(); + const { port } = server; + expect((await send(port, "POST", "/json", { body: '{"a":1}' })).body.toString()).toBe('{"a":1}'); + expect((await send(port, "POST", "/json", { body: "{nope" })).status).toBe(400); + expect((await send(port, "POST", "/json", { body: JSON.stringify({ long: "x".repeat(40) }) })).status).toBe(413); + const chunked = await send(port, "POST", "/body", { + headers: { "transfer-encoding": "chunked" }, + body: "y".repeat(40), + }); + expect(chunked.status).toBe(413); + expect((await send(port, "POST", "/body", { body: "y".repeat(10) })).body.toString()).toBe("10"); + expect(server.errors).toEqual([]); + await server.close(); + }); + + // Sol's and Astra's build reviews: an unfinished chunked upload past the limit got no answer. + it("answers 413 before a chunked upload past the limit has ended", async () => { + const server = await reading(); + const request = httpRequest({ + host: "127.0.0.1", + port: server.port, + path: "/json", + method: "POST", + headers: { "transfer-encoding": "chunked" }, + agent: false, + }); + request.on("error", () => {}); + const status = new Promise((resolve) => { + request.on("response", (response) => { + response.resume(); + resolve(response.statusCode ?? 0); + }); + }); + request.write(`"${"x".repeat(40)}`); + expect(await status).toBe(413); + request.destroy(); + await server.close(); + }); + + it("drains a small body nobody read and keeps the connection; a large one closes it", async () => { + const server = await served({ + "@app/a": contributing({ + routes: { "POST /ignore": { public: true, handler: async () => ({ kind: "json", value: 1 }) } }, + }), + }); + const agent = new Agent({ keepAlive: true, maxSockets: 1 }); + const post = (body: string) => + new Promise<{ connection: string | undefined; socket: unknown }>((resolve, reject) => { + const request = httpRequest( + { host: "127.0.0.1", port: server.port, method: "POST", path: "/ignore", agent }, + (response) => { + response.resume(); + response.on("end", () => + resolve({ connection: response.headers.connection, socket: request.socket }), + ); + }, + ); + request.on("error", reject); + request.end(body); + }); + const first = await post("x".repeat(1000)); + const second = await post("x".repeat(1000)); + expect(first.connection).toBe("keep-alive"); + expect(second.socket).toBe(first.socket); + expect((await post("x".repeat(100_000))).connection).toBe("close"); + agent.destroy(); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/response.test.ts b/packages/chord/test/next-node/http/response.test.ts new file mode 100644 index 000000000..8913f459b --- /dev/null +++ b/packages/chord/test/next-node/http/response.test.ts @@ -0,0 +1,254 @@ +// Tests of responses (slice 10 §2, D5, R6; security baseline (4), (6)): every kind written by a running server, the +// response's headers over the default ones (an invalid one reported, 500), ranges (206, 416, multi-range), `HEAD`, +// attachments and inline bodies, safe download names, bodies without `size`, a short or throwing read, a disconnect +// stopping the reads. + +import { request as httpRequest } from "node:http"; +import { describe, expect, it } from "vitest"; +import type { Ctx } from "../../../src/next/host/index.ts"; +import type { HttpResponse } from "../../../src/next-node/http/index.ts"; +import { contributing, send, served } from "./support.ts"; + +/** A server answering `GET /r` with `response`. */ +async function answering(response: HttpResponse) { + return served({ "@app/r": contributing({ routes: { "GET /r": { public: true, handler: async () => response } } }) }); +} + +/** A `bytes` response over `data` that records its reads. */ +function bytesOf(data: Buffer, extra: Partial> = {}) { + const reads: [number, number][] = []; + const response: HttpResponse = { + kind: "bytes", + mime: "application/octet-stream", + size: data.length, + read: async (offset, length) => { + reads.push([offset, length]); + return data.subarray(offset, offset + length); + }, + ...extra, + }; + return { response, reads }; +} + +describe("descriptions", () => { + it("writes json, html, redirect and error, the default headers first, the response's replacing same-named ones", async () => { + const cases: [HttpResponse, number, string, string | undefined][] = [ + [{ kind: "json", value: { a: 1 } }, 200, '{"a":1}', "application/json; charset=utf-8"], + [ + { kind: "html", body: "

x

", status: 201, headers: { "content-security-policy": "own" } }, + 201, + "

x

", + "text/html; charset=utf-8", + ], + [{ kind: "redirect", to: "/next", status: 303 }, 303, "", undefined], + [{ kind: "error", status: 418 }, 418, "I'm a Teapot", "text/plain; charset=utf-8"], + [{ kind: "error", status: 409, message: "taken" }, 409, "taken", "text/plain; charset=utf-8"], + ]; + for (const [response, status, body, type] of cases) { + const server = await answering(response); + const answer = await send(server.port, "GET", "/r"); + expect([answer.status, answer.body.toString(), answer.headers["content-type"]]).toEqual([status, body, type]); + expect([answer.headers["cache-control"], answer.headers["x-content-type-options"]]).toEqual([ + "no-store", + "nosniff", + ]); + expect(answer.headers["content-security-policy"]).toMatch( + response.headers === undefined ? /^default-src 'none'/ : /^own$/, + ); + if (response.kind === "redirect") expect(answer.headers.location).toBe("/next"); + await server.close(); + } + }); + + it("answers 500 for a header with a CR or LF and a value that is no JSON, reported response_failed", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /header": { + public: true, + handler: async () => ({ kind: "json", value: 1, headers: { "X-Split": "a\r\nSet-Cookie: x=1" } }), + }, + "GET /bigint": { public: true, handler: async () => ({ kind: "json", value: { n: 1n } as never }) }, + }, + }), + }); + for (const path of ["/header", "/bigint"]) { + const answer = await send(server.port, "GET", path); + expect([answer.status, answer.headers["set-cookie"], answer.headers["x-content-type-options"]]).toEqual([ + 500, + undefined, + "nosniff", + ]); + } + expect(server.errors).toMatchObject([ + { kind: "response_failed", pattern: "GET /header" }, + { kind: "response_failed", pattern: "GET /bigint" }, + ]); + await server.close(); + }); +}); + +describe("bytes", () => { + const data = Buffer.from("0123456789".repeat(60_000)); + + it("sends Content-Length, reads 256 KiB pieces, answers a single range 206 and an unsatisfiable one 416", async () => { + const { response, reads } = bytesOf(data, { ranges: true }); + const server = await answering(response); + const whole = await send(server.port, "GET", "/r"); + expect([whole.status, whole.headers["content-length"], whole.headers["accept-ranges"]]).toEqual([ + 200, + "600000", + "bytes", + ]); + expect(whole.body.equals(data)).toBe(true); + expect(reads).toEqual([ + [0, 262144], + [262144, 262144], + [524288, 75712], + ]); + const range = await send(server.port, "GET", "/r", { headers: { range: "bytes=10-19" } }); + expect([range.status, range.body.toString(), range.headers["content-range"]]).toEqual([ + 206, + "0123456789", + "bytes 10-19/600000", + ]); + const suffix = await send(server.port, "GET", "/r", { headers: { range: "bytes=-3" } }); + expect([suffix.status, suffix.body.toString()]).toEqual([206, "789"]); + const past = await send(server.port, "GET", "/r", { headers: { range: "bytes=600000-" } }); + expect([past.status, past.headers["content-range"]]).toEqual([416, "bytes */600000"]); + const multi = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1,5-6" } }); + expect([multi.status, multi.body.length]).toEqual([200, 600000]); + await server.close(); + }); + + it("ignores ranges unless asked; HEAD sends headers only and reads nothing", async () => { + const { response, reads } = bytesOf(data); + const server = await answering(response); + const ranged = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1" } }); + expect([ranged.status, ranged.headers["accept-ranges"]]).toEqual([200, undefined]); + reads.length = 0; + const head = await send(server.port, "HEAD", "/r"); + expect([head.status, head.headers["content-length"], head.body.length, reads]).toEqual([200, "600000", 0, []]); + await server.close(); + }); + + it("names a download with an ASCII fallback and the UTF-8 name, control and bidi characters dropped", async () => { + const server = await answering(bytesOf(Buffer.from("x"), { download: 'Grüße "1"\r\n\u202etxt.exe' }).response); + const answer = await send(server.port, "GET", "/r"); + expect(answer.headers["content-disposition"]).toBe( + `attachment; filename="Gr__e _1_txt.exe"; filename*=UTF-8''Gr%C3%BC%C3%9Fe%20%221%22txt.exe`, + ); + await server.close(); + }); + + it("is an attachment unless inline; an inline one gets a sandbox CSP", async () => { + const attached = await answering(bytesOf(Buffer.from("x")).response); + expect((await send(attached.port, "GET", "/r")).headers["content-disposition"]).toBe("attachment"); + await attached.close(); + const shown = await answering(bytesOf(Buffer.from("x"), { inline: true }).response); + const answer = await send(shown.port, "GET", "/r"); + expect(answer.headers["content-disposition"]).toBe("inline"); + expect(answer.headers["content-security-policy"]).toMatch(/; sandbox$/); + await shown.close(); + }); + + it("refuses a foreign request (another site's) 403, reading nothing", async () => { + const { response, reads } = bytesOf(Buffer.from("secret")); + const server = await answering(response); + const foreign = await send(server.port, "GET", "/r", { headers: { "sec-fetch-site": "cross-site" } }); + expect([foreign.status, reads]).toEqual([403, []]); + expect((await send(server.port, "GET", "/r", { headers: { "sec-fetch-site": "same-origin" } })).status).toBe(200); + await server.close(); + }); + + it("without a size: chunked, read with growing offsets until an empty chunk, no ranges (R6)", async () => { + const pieces = [Buffer.from("ab"), Buffer.from("cde"), Buffer.alloc(0)]; + const offsets: number[] = []; + const server = await answering({ + kind: "bytes", + mime: "text/plain", + ranges: true, + read: async (offset) => { + offsets.push(offset); + return pieces.shift()!; + }, + }); + const answer = await send(server.port, "GET", "/r", { headers: { range: "bytes=0-1" } }); + expect([ + answer.status, + answer.body.toString(), + answer.headers["transfer-encoding"], + answer.headers["accept-ranges"], + ]).toEqual([200, "abcde", "chunked", undefined]); + expect(offsets).toEqual([0, 2, 5]); + await server.close(); + }); + + it("destroys the connection on a short or a throwing read, reported response_failed", async () => { + const server = await served({ + "@app/r": contributing({ + routes: { + "GET /short": { + public: true, + handler: async () => ({ + kind: "bytes", + mime: "text/plain", + size: 10, + read: async () => Buffer.from("abc"), + }), + }, + // Astra's build review: a throwing read hung the response, unreported. + "GET /throws": { + public: true, + handler: async () => ({ + kind: "bytes", + mime: "text/plain", + size: 10, + read: async () => { + throw new Error("read failed"); + }, + }), + }, + }, + }), + }); + await expect(send(server.port, "GET", "/short")).rejects.toThrow(); + await expect(send(server.port, "GET", "/throws")).rejects.toThrow(); + expect(server.errors).toMatchObject([ + { kind: "response_failed", pattern: "GET /short" }, + { kind: "response_failed", pattern: "GET /throws", error: { message: "read failed" } }, + ]); + await server.close(); + }); + + it("stops reading when the client disconnects: the read's context is aborted", async () => { + let reads = 0; + let signal: AbortSignal | undefined; + const server = await answering({ + kind: "bytes", + mime: "application/octet-stream", + size: 1024 * 1024 * 1024, + read: async (_offset, length, ctx: Ctx) => { + reads++; + signal = ctx.abortSignal; + return Buffer.alloc(length); + }, + }); + await new Promise((resolve) => { + const request = httpRequest({ host: "127.0.0.1", port: server.port, path: "/r", agent: false }, (response) => { + response.once("data", () => { + request.destroy(); + resolve(); + }); + }); + request.end(); + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + const after = reads; + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(signal?.aborted).toBe(true); + expect(reads).toBe(after); + expect(after).toBeLessThan(1000); + await server.close(); + }); +}); diff --git a/packages/chord/test/next-node/http/router.test.ts b/packages/chord/test/next-node/http/router.test.ts new file mode 100644 index 000000000..8bcff127e --- /dev/null +++ b/packages/chord/test/next-node/http/router.test.ts @@ -0,0 +1,235 @@ +// Tests of the router (slice 10 §2, D2, D8): patterns, decoding (one rule for `:name` and rest segments), precedence, +// `HEAD` per path, each route's own param names, route options, conflicts, wraps resolved by phase, contributor order, +// scope and route, allowed hosts, framers, relays, CSP sources, sockets. + +import { describe, expect, it } from "vitest"; +import type { RegistryConflict, RegistryTable } from "../../../src/next/registries/index.ts"; +import { + composeRouter, + type HttpContribution, + type HttpRoute, + type HttpWrap, + type RouteMatch, + type Router, +} from "../../../src/next-node/http/router.ts"; + +/** A route named `name` (its handler answers its name). */ +const route = + (name: string): HttpRoute => + async () => ({ kind: "html", body: name }); + +/** The router of `entries` (plugin, contribution), with the conflicts it reported. */ +function compose(...entries: [string, HttpContribution][]): { router: Router; conflicts: RegistryConflict[] } { + const conflicts: RegistryConflict[] = []; + const table: RegistryTable = entries.map(([plugin, value]) => ({ plugin, version: 1, value })); + return { router: composeRouter(table, { report: (conflict) => conflicts.push(conflict) }), conflicts }; +} + +/** The matched route's pattern and params, or what `route` answered. */ +function matched(router: Router, method: string, path: string) { + const match = router.route(method, path); + if (match === undefined || match === "bad_path") return match; + return { pattern: match.pattern, params: match.params }; +} + +describe("patterns and precedence (D8)", () => { + const { router } = compose([ + "@app/a", + { + routes: { + "GET /notes": route("list"), + "GET /notes/:id": route("one"), + "GET /notes/new": route("new"), + "GET /notes/:id/export": route("export"), + "GET /files/*": route("files"), + "GET /files/special/:name": route("special"), + "GET /": route("root"), + }, + }, + ]); + + it("matches exact segments, :name and a trailing *, the most specific from the left", () => { + expect(matched(router, "GET", "/notes")).toEqual({ pattern: "GET /notes", params: {} }); + expect(matched(router, "GET", "/notes/new")).toEqual({ pattern: "GET /notes/new", params: {} }); + expect(matched(router, "GET", "/notes/7")).toEqual({ pattern: "GET /notes/:id", params: { id: "7" } }); + expect(matched(router, "GET", "/notes/7/export")).toMatchObject({ params: { id: "7" } }); + expect(matched(router, "GET", "/files/special/x")).toEqual({ + pattern: "GET /files/special/:name", + params: { name: "x" }, + }); + // The exact branch fails deeper: the rest takes it. + expect(matched(router, "GET", "/files/special/x/y")).toEqual({ + pattern: "GET /files/*", + params: { "*": "special/x/y" }, + }); + expect(matched(router, "GET", "/")).toEqual({ pattern: "GET /", params: {} }); + }); + + it("a /* matches its prefix with an empty rest; :name needs a non-empty segment", () => { + expect(matched(router, "GET", "/files")).toEqual({ pattern: "GET /files/*", params: { "*": "" } }); + expect(matched(router, "GET", "/notes/")).toBeUndefined(); + }); + + // Opus's build review F4, security audit H5: `%2F` in `:name` reached a file path. + it("percent-decodes :name and rest segments; one decoding to /, \\, NUL, . or .. is a bad path", () => { + expect(matched(router, "GET", "/notes/a%20b")).toMatchObject({ params: { id: "a b" } }); + expect(matched(router, "GET", "/files/a%20b/c")).toMatchObject({ params: { "*": "a b/c" } }); + for (const bad of ["a%2Fb", "..%2F..%2Fsecret", "a%5Cb", "a%00b", "%2e%2e", "%2E", ".", ".."]) { + expect(router.route("GET", `/notes/${bad}`)).toBe("bad_path"); + expect(router.route("GET", `/files/x/${bad}`)).toBe("bad_path"); + } + expect(router.route("GET", "/notes/%E0%A4%A")).toBe("bad_path"); + }); + + it("falls back to the GET route for HEAD; another method or path has none", () => { + expect(matched(router, "HEAD", "/notes")).toEqual({ pattern: "GET /notes", params: {} }); + expect(router.route("POST", "/notes")).toBeUndefined(); + expect(router.route("GET", "/nope")).toBeUndefined(); + }); + + // Opus's build review F2, Astra's: one explicit HEAD route anywhere hid every GET fallback. + it("falls back to GET per path when another path has an explicit HEAD route", () => { + const { router } = compose([ + "@app/a", + { routes: { "HEAD /download": route("head"), "GET /download": route("get"), "GET /notes": route("notes") } }, + ]); + expect(matched(router, "HEAD", "/download")).toMatchObject({ pattern: "HEAD /download" }); + expect(matched(router, "HEAD", "/notes")).toMatchObject({ pattern: "GET /notes" }); + }); + + // Opus's build review F3, Astra's: the first route's param name was used for every route at that position. + it("names each route's params by its own pattern", () => { + const { router } = compose(["@app/a", { routes: { "GET /:id/a": route("a"), "GET /:name/b": route("b") } }]); + expect(matched(router, "GET", "/v/a")).toEqual({ pattern: "GET /:id/a", params: { id: "v" } }); + expect(matched(router, "GET", "/v/b")).toEqual({ pattern: "GET /:name/b", params: { name: "v" } }); + }); + + it("takes a route with options: page, crossSite and public (a cross-site route is public: always anonymous)", () => { + const { router } = compose([ + "@app/a", + { + routes: { + "GET /p": { handler: route("p"), page: true }, + "POST /hook": { handler: route("h"), crossSite: true }, + "GET /open": { handler: route("o"), public: true }, + "GET /closed": route("c"), + }, + sockets: { "/s": { handler: () => {}, public: true }, "/t": () => {} }, + }, + ]); + expect(router.route("GET", "/p")).toMatchObject({ page: true, crossSite: false, public: false }); + expect(router.route("POST", "/hook")).toMatchObject({ page: false, crossSite: true, public: true }); + expect(router.route("GET", "/open")).toMatchObject({ public: true }); + expect(router.route("GET", "/closed")).toMatchObject({ public: false }); + expect(router.socket("/s")).toMatchObject({ public: true }); + expect(router.socket("/t")).toMatchObject({ public: false }); + expect(router.route("GET", "/notes")).toBeUndefined(); + }); +}); + +describe("conflicts", () => { + it("reports one pattern from two contributions (also with other param names); the first by plugin name wins", () => { + const { router, conflicts } = compose( + ["@app/a", { routes: { "GET /x/:id": route("a") } }], + ["@app/b", { routes: { "GET /x/:key": route("b"), BAD: route("bad") }, sockets: { nope: () => {} } }], + ); + expect((router.route("GET", "/x/1") as RouteMatch).plugin).toBe("@app/a"); + expect(conflicts).toEqual([ + { message: 'route "GET /x/:key" is contributed twice', plugins: ["@app/a", "@app/b"] }, + { message: 'route "BAD" is no "METHOD /pattern"', plugins: ["@app/b"] }, + { message: 'socket "nope" is no pattern', plugins: ["@app/b"] }, + ]); + }); + + it("refuses a route for a method a Web Request cannot carry, reported", () => { + const { router, conflicts } = compose([ + "@app/a", + { routes: { "TRACE /x": route("t"), "CONNECT /y": route("c") } }, + ]); + expect(router.route("TRACE", "/x")).toBeUndefined(); + expect(conflicts).toEqual([ + { message: 'route "TRACE /x": TRACE is served for no resource (501)', plugins: ["@app/a"] }, + { message: 'route "CONNECT /y": CONNECT is served for no resource (501)', plugins: ["@app/a"] }, + ]); + }); +}); + +describe("wraps (D2)", () => { + const wrap = (name: string, phase: HttpWrap["phase"], extra: Partial = {}): HttpWrap => ({ + name, + phase, + wrap: (request, next) => next(request), + ...extra, + }); + + it("orders by phase (outermost first), then contributor order; own wraps around their plugin's routes only", () => { + const { router } = compose( + [ + "@app/a", + { + routes: { "GET /a": route("a") }, + wraps: [wrap("a-transform", "transform"), wrap("a-observe", "observe")], + }, + ], + [ + "@app/b", + { + routes: { "GET /b": route("b") }, + wraps: [wrap("b-all", "headers", { scope: "all" }), wrap("b-own", "observe")], + }, + ], + ); + const names = (path: string) => (router.route("GET", path) as RouteMatch).wraps.map(({ wrap }) => wrap.name); + expect(names("/a")).toEqual(["a-observe", "b-all", "a-transform"]); + expect(names("/b")).toEqual(["b-own", "b-all"]); + }); + + it("narrows to one route; a route nobody contributes is a reported conflict", () => { + const { router, conflicts } = compose([ + "@app/a", + { + routes: { "GET /a": route("a"), "GET /b": route("b") }, + wraps: [wrap("only-a", "observe", { route: "GET /a" }), wrap("ghost", "observe", { route: "GET /c" })], + }, + ]); + expect((router.route("GET", "/a") as RouteMatch).wraps.map(({ wrap }) => wrap.name)).toEqual(["only-a"]); + expect((router.route("GET", "/b") as RouteMatch).wraps).toEqual([]); + expect(conflicts).toEqual([ + { message: 'wrap "ghost" names route "GET /c", which nobody contributes', plugins: ["@app/a"] }, + ]); + }); +}); + +describe("hosts and sockets", () => { + it("collects hosts lowercased, framers, relays and CSP sources; matches sockets with their largest frame", () => { + const terminal = () => {}; + const relay = { name: "radius", verify: () => undefined }; + const contribution: HttpContribution = { + hosts: ["Notes.Example.com"], + sockets: { "/terminal/:id": terminal }, + maxPayload: 10, + framers: ["https://frame.example.com"], + relays: [relay], + csp: { "img-src": ["https://cdn.example.com"] }, + }; + const { router } = compose( + ["@app/a", contribution], + ["@app/b", { hosts: ["b.example.com"], csp: { "img-src": ["data:"] } }], + ); + expect([...router.hosts]).toEqual(["notes.example.com", "b.example.com"]); + expect([[...router.framers], router.relays, [...router.csp]]).toEqual([ + ["https://frame.example.com"], + [relay], + [["img-src", ["https://cdn.example.com", "data:"]]], + ]); + expect(router.socket("/terminal/7")).toMatchObject({ + pattern: "/terminal/:id", + plugin: "@app/a", + params: { id: "7" }, + maxPayload: 10, + }); + expect(router.socket("/terminal/..")).toBe("bad_path"); + expect(router.socket("/nope")).toBeUndefined(); + expect(router.socketHandlers.has(terminal)).toBe(true); + }); +}); diff --git a/packages/chord/test/next-node/http/support.ts b/packages/chord/test/next-node/http/support.ts new file mode 100644 index 000000000..8a4b82b59 --- /dev/null +++ b/packages/chord/test/next-node/http/support.ts @@ -0,0 +1,204 @@ +// Test support for the HTTP component (slice 10 §2): a server host with the component, the app's configuration and +// authenticator, and plugin facets contributing to `Http`; raw requests (any `Host`, any body) and WebSocket clients +// against its listener. + +import { request as httpRequest, type IncomingHttpHeaders } from "node:http"; +import { WebSocket } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import type { Address } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Facet, type Host, type HostError } from "../../../src/next/host/index.ts"; +import { + type Authenticator, + Authenticators, + createHttp, + Http, + type HttpComponent, + type HttpContribution, + type HttpError, + type HttpOptions, +} from "../../../src/next-node/http/index.ts"; + +/** The principals of these tests. */ +export type TestPrincipal = + | { readonly kind: "server" } + | { readonly kind: "owner" } + | { readonly kind: "guest"; readonly id: string }; + +/** A running server: its host, its component, its port once loaded, what both reported. */ +export type Served = { + readonly host: Host; + readonly http: HttpComponent; + /** The listener's port (after `load`). */ + readonly port: number; + readonly errors: HttpError[]; + readonly hostErrors: HostError[]; + /** The core facets and the plugins, as loaded. */ + facets(plugins?: Readonly>>): { name: string; facet: Facet }[]; + close(): Promise; +}; + +/** A plugin facet contributing `contribution` to `Http`. */ +export function contributing(contribution: HttpContribution): Facet { + return defineFacet((_host, self) => self.contribute(Http, contribution)); +} + +/** + * Builds the component on a new server host (listening on loopback, a free port), then loads it with the app's + * authenticator (`@app/auth`: the `x-principal` header names one, else anonymous) and `plugins` (by name) unless + * `load` is false; `config` overrides the settings (`listen: undefined`: no listener). + */ +export async function served( + plugins: Readonly>> = {}, + options: { + readonly config?: Omit; + readonly authenticate?: Authenticator; + readonly load?: boolean; + readonly readyTimeout?: number; + readonly address?: Address; + } = {}, +): Promise }> { + const errors: HttpError[] = []; + const hostErrors: HostError[] = []; + const host = createHost({ + principal: { kind: "server" }, + onError: (error) => hostErrors.push(error), + ...(options.readyTimeout === undefined ? {} : { readyTimeout: options.readyTimeout }), + ...(options.address === undefined ? {} : { address: options.address }), + }); + const config: Omit = { listen: { host: "127.0.0.1", port: 0 }, ...options.config }; + const http = createHttp(host, { ...config, onError: (error) => errors.push(error) }, context); + const authenticate = options.authenticate ?? ((request) => principalOf(request.headers)); + const core = [ + { name: "chord.http", facet: http.facet }, + { + name: "@app/auth", + facet: defineFacet((_host, self) => self.contribute(Authenticators, authenticate)), + }, + ]; + const facets = (more = plugins) => [...core, ...Object.entries(more).map(([name, facet]) => ({ name, facet }))]; + let port = 0; + const load = async (): Promise => { + await host.load(facets(), context); + if (config.listen !== undefined) port = (await http.listening(context)).port; + }; + if (options.load !== false) await load(); + return { + host, + http, + get port() { + return port; + }, + errors, + hostErrors, + facets, + load, + close: async () => { + await http.close(context); + await host.close(context); + }, + }; +} + +/** The principal an `x-principal` header names: `owner`, `guest:`; none without it. */ +function principalOf(headers: Headers): TestPrincipal | undefined { + const named = headers.get("x-principal"); + if (named === "owner") return { kind: "owner" }; + if (named?.startsWith("guest:")) return { kind: "guest", id: named.slice(6) }; + return undefined; +} + +/** A response as a test reads it. */ +export type Answer = { readonly status: number; readonly headers: IncomingHttpHeaders; readonly body: Buffer }; + +/** + * A request to `port` with raw control: `Host` defaults to `127.0.0.1:`; `body` is sent as is; `signal` aborts it. + */ +export function send( + port: number, + method: string, + path: string, + options: { + readonly headers?: Readonly>; + readonly body?: string | Buffer; + readonly agent?: false; + } = {}, +): Promise { + return new Promise((resolve, reject) => { + const request = httpRequest( + { + host: "127.0.0.1", + port, + method, + path, + agent: false, + headers: { host: `127.0.0.1:${port}`, ...options.headers }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => + resolve({ status: response.statusCode ?? 0, headers: response.headers, body: Buffer.concat(chunks) }), + ); + response.on("error", reject); + }, + ); + request.on("error", reject); + request.end(options.body); + }); +} + +/** A WebSocket client to `path` on `port` (`Host` loopback), with `headers`; resolves once open, rejects when refused. */ +export function socketTo( + port: number, + path: string, + headers: Readonly> = {}, + options: { readonly maxPayload?: number } = {}, +): Promise { + return new Promise((resolve, reject) => { + const socket = new WebSocket(`ws://127.0.0.1:${port}${path}`, { headers, ...options }); + // Frames sent at once by the server arrive with the upgrade: queued from the start. + inboxes.set(socket, messagesOf(socket)); + socket.once("open", () => resolve(socket)); + socket.once("unexpected-response", (_request, response) => { + reject(new Error(`refused ${response.statusCode}`)); + socket.terminate(); + }); + socket.once("error", reject); + }); +} + +/** The messages of every socket `socketTo` opened, queued from its start. */ +const inboxes = new WeakMap }>(); + +/** The next text message of a socket `socketTo` opened. */ +export function nextMessage(socket: WebSocket): Promise { + return inboxes.get(socket)!.next(); +} + +/** The text messages `socket` receives, from now on, in order: `next()` waits for the next one. */ +export function messagesOf(socket: WebSocket): { next(): Promise } { + const queued: string[] = []; + const waiting: ((text: string) => void)[] = []; + socket.on("message", (data) => { + const text = String(data); + const waiter = waiting.shift(); + if (waiter === undefined) queued.push(text); + else waiter(text); + }); + return { + next: () => { + const text = queued.shift(); + return text === undefined ? new Promise((resolve) => waiting.push(resolve)) : Promise.resolve(text); + }, + }; +} + +/** The close code and reason of `socket`'s close. */ +export function closeOf(socket: WebSocket): Promise<{ readonly code: number; readonly reason: string }> { + return new Promise((resolve) => { + if (socket.readyState === socket.CLOSED) resolve({ code: 1006, reason: "" }); + socket.once("close", (code, reason) => resolve({ code, reason: reason.toString() })); + }); +} + +export { context }; diff --git a/packages/chord/test/next-node/http/upgrades.test.ts b/packages/chord/test/next-node/http/upgrades.test.ts new file mode 100644 index 000000000..5c0127fd0 --- /dev/null +++ b/packages/chord/test/next-node/http/upgrades.test.ts @@ -0,0 +1,500 @@ +// Tests of upgraded connections (slice 10 §2, D6, Q6; decisions "allowed hosts only through contributions", "plugin +// sockets get a heartbeat"): the links endpoint (`host.accept`, options per socket, refused sockets, the 1 MiB frame +// default, the output cap), plugin sockets (handed to their handler, an oversized frame closing only that socket, a +// throwing handler), connections terminated at once when their socket handler or host leaves (a raw peer can act no +// more), an upgrade checked again after authenticating, the heartbeat, close bounded for a peer that never answers, +// and nothing kept of a closed socket or its `ws` server. + +import { once } from "node:events"; +import { connect, type Socket } from "node:net"; +import { describe, expect, it, vi } from "vitest"; +import { WebSocket, WebSocketServer } from "ws"; +import { arr, hostKind, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { webSocketLink } from "../../../src/next/link/index.ts"; +import { Http, type HttpContribution } from "../../../src/next-node/http/index.ts"; +import { limitedSocket } from "../../../src/next-node/http/upgrades.ts"; +import { expectCollected, settledHeap } from "../../next/support.ts"; +import { closeOf, context, contributing, nextMessage, send, served, socketTo, type TestPrincipal } from "./support.ts"; + +const Server = hostKind("server"); + +/** Who the server sees calling: served at the server's address. */ +const Who = remote("test.who", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { methods: { who: method(undefined, str()) } } }, +}); + +const serveWho = defineFacet((host) => + host.serve(Who, () => ({ + variant: "main", + members: { who: async (ctx) => JSON.stringify(ctx.principal ?? null) }, + })), +); + +/** A plugin socket that echoes text, with its handler's request params and principal sent first. */ +const echo = contributing({ + sockets: { + "/echo/:room": (socket, request, ctx) => { + socket.send(`${request.params.room} as ${ctx.principal?.kind ?? "nobody"}`); + socket.on("message", (data) => socket.send(String(data))); + }, + }, + maxPayload: 64, +}); + +describe("the links endpoint", () => { + it("accepts a link as the authenticated principal, with the options links.options gave that socket", async () => { + const options: unknown[] = []; + const server = await served( + { "@app/who": serveWho }, + { + address: Server(), + config: { + links: { + path: "/chord", + options: (request, principal) => { + options.push([request.path, principal]); + return { maxPayload: 1024 * 1024 }; + }, + }, + }, + }, + ); + const tab = createHost({ onError: () => {} }); + await tab.load([], context); + // The link wraps the socket at once: the server's `hello` comes with the upgrade. + const socket = new WebSocket(`ws://127.0.0.1:${server.port}/chord`, { headers: { "x-principal": "owner" } }); + tab.connect(webSocketLink(socket as never), { principal: { kind: "server" } }, context); + const who = await tab.get(Who, {}, context); + expect(await who.who(context)).toBe('{"kind":"owner"}'); + expect(options).toEqual([["/chord", { kind: "owner" }]]); + await tab.close(context); + await server.close(); + }); + + it("refuses a socket without a principal (401), from another Origin or Host, foreign, or whose links.options throws (403)", async () => { + const server = await served( + { "@app/hosts": contributing({ hosts: ["a.example.com"] }) }, + { + config: { + links: { + path: "/chord", + options: (request) => { + if (request.headers.has("x-fail")) throw new Error("options"); + return {}; + }, + }, + }, + }, + ); + const { port } = server; + const owner = { "x-principal": "owner" }; + // Nobody: the links endpoint needs a principal (deny by default, slice 13). + await expect(socketTo(port, "/chord")).rejects.toThrow("refused 401"); + const refusals = [ + socketTo(port, "/chord", { ...owner, origin: "http://evil.com" }), + socketTo(port, "/chord", { ...owner, origin: "null" }), + socketTo(port, "/chord", { ...owner, origin: `HTTP://127.0.0.1:${port}` }), + socketTo(port, "/chord", { ...owner, host: "evil.com" }), + socketTo(port, "/chord", { ...owner, "sec-fetch-site": "cross-site" }), + socketTo(port, "/chord", { ...owner, "x-fail": "1" }), + ]; + for (const refusal of refusals) await expect(refusal).rejects.toThrow("refused 403"); + const same = await socketTo(port, "/chord", { ...owner, origin: `http://127.0.0.1:${port}` }); + same.close(); + expect(server.errors).toEqual([{ kind: "hook_failed", hook: "links.options", error: new Error("options") }]); + await server.close(); + }); + + it("calls links.opened for each accepted link and what it returned when that link closes (slice 11)", async () => { + const events: string[] = []; + const server = await served( + {}, + { + config: { + links: { + path: "/chord", + opened: (request, principal) => { + events.push(`opened ${request.path} ${principal.kind}`); + return () => events.push("closed"); + }, + }, + }, + }, + ); + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + for (let index = 0; index < 100 && events.length === 0; index++) await new Promise((r) => setTimeout(r, 5)); + expect(events).toEqual(["opened /chord owner"]); + socket.close(); + for (let index = 0; index < 100 && events.length === 1; index++) await new Promise((r) => setTimeout(r, 5)); + expect(events).toEqual(["opened /chord owner", "closed"]); + await server.close(); + }); + + it("takes frames of at most 1 MiB unless links.options says otherwise (Opus's audit H2)", async () => { + const server = await served({}, { config: { links: { path: "/chord" } } }); + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.send(Buffer.alloc(1024 * 1024 + 1)); + expect((await closed).code).toBe(1009); + await server.close(); + }); + + it("cuts off a client that stopped reading: past 4 MiB unsent beyond the frame being sent", () => { + const sent: number[] = []; + let terminated = 0; + const fake = { + OPEN: 1, + readyState: 1, + bufferedAmount: 0, + binaryType: "nodebuffer", + send: (data: Uint8Array) => void sent.push(data.byteLength), + terminate: () => void terminated++, + }; + const limited = limitedSocket(fake as never); + limited.send(new Uint8Array(10)); + fake.bufferedAmount = 5 * 1024 * 1024; + limited.send(new Uint8Array(10)); + expect([sent, terminated]).toEqual([[10], 1]); + }); +}); + +describe("plugin sockets", () => { + it("hands the socket and its request to the handler in a scope with its principal; no principal is refused", async () => { + const server = await served({ "@app/echo": echo }); + await expect(socketTo(server.port, "/echo/lobby")).rejects.toThrow("refused 401"); + await expect(socketTo(server.port, "/nope", { "x-principal": "owner" })).rejects.toThrow("refused 404"); + const socket = await socketTo(server.port, "/echo/lobby", { "x-principal": "owner" }); + expect(await nextMessage(socket)).toBe("lobby as owner"); + socket.send("hi"); + expect(await nextMessage(socket)).toBe("hi"); + socket.close(); + await server.close(); + }); + + it("accepts a public socket without a principal, its scope anonymous (deny by default, slice 13 §2)", async () => { + const lobby = contributing({ + sockets: { + "/lobby": { + public: true, + handler: (socket, _request, ctx) => socket.send(`as ${ctx.principal?.kind ?? "nobody"}`), + }, + }, + }); + const server = await served({ "@app/lobby": lobby }); + const socket = await socketTo(server.port, "/lobby"); + expect(await nextMessage(socket)).toBe("as nobody"); + socket.close(); + await server.close(); + }); + + it("an oversized frame closes only that socket (1009), never the server", async () => { + const server = await served({ "@app/echo": echo }); + const [big, other] = await Promise.all([ + socketTo(server.port, "/echo/a", { "x-principal": "owner" }), + socketTo(server.port, "/echo/b", { "x-principal": "owner" }), + ]); + await Promise.all([nextMessage(big), nextMessage(other)]); + const closed = closeOf(big); + big.send("x".repeat(100)); + expect((await closed).code).toBe(1009); + other.send("still here"); + expect(await nextMessage(other)).toBe("still here"); + other.close(); + await server.close(); + }); + + it("a throwing handler is reported socket_failed and closes its socket (1011)", async () => { + const server = await served({ + "@app/bad": contributing({ + sockets: { + "/bad": () => { + throw new Error("handler"); + }, + }, + }), + }); + const socket = await socketTo(server.port, "/bad", { "x-principal": "owner" }); + expect((await closeOf(socket)).code).toBe(1011); + expect(server.errors).toMatchObject([{ kind: "socket_failed", pattern: "/bad", plugin: "@app/bad" }]); + await server.close(); + }); +}); + +describe("revocation", () => { + it("terminates a plugin's sockets at once when a reload removes its contribution, sealing their scopes; others stay (Q6)", async () => { + let signal: AbortSignal | undefined; + const removed = contributing({ + sockets: { + "/removed": (_socket, _request, ctx) => { + signal = ctx.abortSignal; + }, + }, + }); + const other = contributing({ sockets: { "/other": () => {} } }); + const server = await served({ "@app/removed": removed, "@app/other": other }); + const doomed = await socketTo(server.port, "/removed", { "x-principal": "owner" }); + const kept = await socketTo(server.port, "/other", { "x-principal": "owner" }); + const closed = closeOf(doomed); + await server.host.reload(server.facets({ "@app/other": other }), context); + expect(signal?.aborted).toBe(true); + expect((await closed).code).toBe(1006); + expect(kept.readyState).toBe(kept.OPEN); + kept.close(); + await server.close(); + }); + + it("terminates links and sockets through a host no longer contributed; loopback ones stay", async () => { + const server = await served( + { "@app/echo": echo, "@app/hosts": contributing({ hosts: ["a.example.com"] }) }, + { config: { links: { path: "/chord" } } }, + ); + const through = { "x-principal": "owner", host: "a.example.com" }; + const link = await socketTo(server.port, "/chord", through); + const socket = await socketTo(server.port, "/echo/a", through); + const loopback = await socketTo(server.port, "/echo/b", { "x-principal": "owner" }); + const closes = [closeOf(link), closeOf(socket)]; + await server.host.reload(server.facets({ "@app/echo": echo }), context); + expect((await Promise.all(closes)).map(({ code }) => code)).toEqual([1006, 1006]); + expect(loopback.readyState).toBe(loopback.OPEN); + expect((await send(server.port, "GET", "/", { headers: { host: "a.example.com" } })).status).toBe(403); + loopback.close(); + await server.close(); + }); + + it("keeps a writable contribution's own sockets when only its hosts change", async () => { + const Tunnel = local("app.tunnel", { version: 1, params: {}, members: { name: method(arr(str())) } }); + const handler = (): void => {}; + const tunnel = defineFacet((host) => + host.serve(Tunnel, (_, self) => { + const contribution = self.writable({ hosts: [], sockets: { "/own": handler } }); + self.contribute(Http, contribution); + return { + name: async (hosts) => { + contribution.set({ hosts, sockets: { "/own": handler } }); + return null; + }, + }; + }), + ); + const server = await served({ "@app/tunnel": tunnel }); + const handle = await server.host.get(Tunnel, {}, context); + const socket = await socketTo(server.port, "/own", { "x-principal": "owner" }); + await handle.name(["t.example.com"], context); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(socket.readyState).toBe(socket.OPEN); + socket.close(); + await server.close(); + }); + + // Astra's security audit A1: a policy close left the peer able to invoke the plugin's handlers. + it("a raw peer whose host or contribution left can send nothing more to the handler", async () => { + const calls: string[] = []; + const sockets = contributing({ + sockets: { "/echo": (socket) => void socket.on("message", (data) => calls.push(String(data))) }, + }); + const tunnel = contributing({ hosts: ["t.example"] }); + const server = await served({ "@app/sockets": sockets, "@app/tunnel": tunnel }); + const throughHost = await rawSocket(server.port, "t.example", "/echo"); + const local = await rawSocket(server.port, `localhost:${server.port}`, "/echo"); + await server.host.reload(server.facets({ "@app/sockets": sockets }), context); + throughHost.write(frame("after the host left")); + await new Promise((resolve) => setTimeout(resolve, 50)); + await server.host.reload(server.facets({}), context); + local.write(frame("after the contribution left")); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(calls).toEqual([]); + throughHost.destroy(); + local.destroy(); + await server.close(); + }); + + // Astra's security audit A2: an upgrade authenticating across the removal of its host was still handed out. + it("checks an upgrade again after authenticating: one whose host left meanwhile is refused", async () => { + const gate = Promise.withResolvers(); + const started = Promise.withResolvers(); + const sockets = contributing({ sockets: { "/echo": () => {} } }); + const server = await served( + { "@app/sockets": sockets, "@app/tunnel": contributing({ hosts: ["t.example"] }) }, + { + authenticate: async () => { + started.resolve(); + await gate.promise; + return { kind: "owner" }; + }, + }, + ); + const opening = socketTo(server.port, "/echo", { host: "t.example" }); + await started.promise; + await server.host.reload(server.facets({ "@app/sockets": sockets }), context); + gate.resolve(); + await expect(opening).rejects.toThrow("refused 403"); + await server.close(); + }); +}); + +describe("cancellation", () => { + // Astra's recheck R1, Sol's N5: a client gone while its upgrade authenticated left that work running. + // Fable's second recheck: only a reset (RST) was seen; a clean disconnect (FIN, a tab closing) was not. + it.each(["ends its connection", "resets its connection"] as const)( + "aborts an upgrade's authentication when its client %s", + async (how) => { + const entered = Promise.withResolvers(); + let signal: AbortSignal | undefined; + const server = await served( + { "@app/s": contributing({ sockets: { "/s": () => {} } }) }, + { + authenticate: async (_request, ctx) => { + signal = ctx.abortSignal; + entered.resolve(); + return new Promise(() => {}); + }, + }, + ); + const socket = connect(server.port, "127.0.0.1"); + socket.on("error", () => {}); + await once(socket, "connect"); + socket.write( + `GET /s HTTP/1.1\r\nHost: 127.0.0.1:${server.port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n`, + ); + await entered.promise; + const aborted = new Promise((resolve) => signal?.addEventListener("abort", resolve, { once: true })); + if (how === "ends its connection") socket.end(); + else socket.resetAndDestroy(); + await aborted; + expect(signal?.aborted).toBe(true); + socket.destroy(); + await server.close(); + }, + ); +}); + +describe("heartbeat", () => { + it("pings a plugin socket after 15 s of silence and terminates it 15 s later without an answer", async () => { + const server = await served({ + "@app/quiet": contributing({ + sockets: { "/quiet": (socket) => void socket.on("message", (data) => socket.send(data)) }, + }), + }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + try { + const silent = new WebSocket(`ws://127.0.0.1:${server.port}/quiet`, { + headers: { "x-principal": "owner" }, + autoPong: false, + }); + const answering = new WebSocket(`ws://127.0.0.1:${server.port}/quiet`, { + headers: { "x-principal": "owner" }, + }); + await Promise.all([once(silent, "open"), once(answering, "open")]); + const pinged = Promise.all([once(silent, "ping"), once(answering, "ping")]); + await vi.advanceTimersByTimeAsync(15_000); + await pinged; + // The answering client's pong travels for real (only timers are fake): an echo after it shows it arrived. + const echoed = once(answering, "message"); + answering.send("after the pong"); + await echoed; + const closed = closeOf(silent); + await vi.advanceTimersByTimeAsync(15_000); + expect((await closed).code).toBe(1006); + expect(answering.readyState).toBe(answering.OPEN); + answering.close(); + } finally { + vi.useRealTimers(); + } + await server.close(); + }); +}); + +describe("close", () => { + // Sol's build review: close waited on a peer that never acknowledged its close frame. + it("terminates a socket that never acknowledges its close at closeTimeout", async () => { + const server = await served( + { "@app/s": contributing({ sockets: { "/s": () => {} } }) }, + { config: { closeTimeout: 20 } }, + ); + const raw = await rawSocket(server.port, `127.0.0.1:${server.port}`, "/s"); + raw.pause(); + const started = Date.now(); + await server.http.close(context); + expect(Date.now() - started).toBeLessThan(500); + raw.destroy(); + await server.close(); + }); +}); + +describe("GC", () => { + // Sol's build review: a `ws` server was kept per frame size for the component's life. + it("keeps no ws server once its socket closed", async () => { + const refs: WeakRef[] = []; + const original = WebSocketServer.prototype.handleUpgrade; + WebSocketServer.prototype.handleUpgrade = function (this: WebSocketServer, ...args: Parameters) { + refs.push(new WeakRef(this)); + return original.apply(this, args); + }; + let maxPayload = 1024; + const server = await served( + {}, + { config: { links: { path: "/chord", options: () => ({ maxPayload: maxPayload++ }) } } }, + ); + try { + for (let i = 0; i < 8; i++) { + const socket = await socketTo(server.port, "/chord", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.close(); + await closed; + } + await settledHeap(); + await expectCollected(...refs); + } finally { + WebSocketServer.prototype.handleUpgrade = original; + await server.close(); + } + }); + + it("keeps nothing of a closed plugin socket", async () => { + const refs: WeakRef[] = []; + const server = await served({ + "@app/kept": contributing({ + sockets: { + "/kept": (socket, request, ctx) => { + refs.push(new WeakRef(socket), new WeakRef(request), new WeakRef(ctx)); + }, + }, + }), + }); + const socket = await socketTo(server.port, "/kept", { "x-principal": "owner" }); + const closed = closeOf(socket); + socket.close(); + await closed; + await new Promise((resolve) => setTimeout(resolve, 50)); + await expectCollected(...refs); + await server.close(); + }); +}); + +/** A raw TCP client upgraded to the plugin socket `path` through `host` as the owner. */ +async function rawSocket(port: number, host: string, path: string): Promise { + const socket = connect(port, "127.0.0.1"); + await once(socket, "connect"); + socket.on("error", () => {}); + socket.write( + `GET ${path} HTTP/1.1\r\nHost: ${host}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Key: AAAAAAAAAAAAAAAAAAAAAA==\r\nSec-WebSocket-Version: 13\r\nx-principal: owner\r\n\r\n`, + ); + const [answer] = await once(socket, "data"); + expect(String(answer)).toContain("101 Switching Protocols"); + return socket; +} + +/** A masked client text frame. */ +function frame(text: string): Buffer { + const bytes = Buffer.from(text); + const mask = Buffer.from([1, 2, 3, 4]); + return Buffer.concat([ + Buffer.from([0x81, 0x80 | bytes.length]), + mask, + Buffer.from(bytes.map((byte, index) => byte ^ mask[index % 4]!)), + ]); +} diff --git a/packages/chord/test/next-node/plugins/build.test.ts b/packages/chord/test/next-node/plugins/build.test.ts new file mode 100644 index 000000000..528efc152 --- /dev/null +++ b/packages/chord/test/next-node/plugins/build.test.ts @@ -0,0 +1,744 @@ +// Tests of building (slice 7c §2): every row of the import table, failures as results, source maps relative to the +// plugin and digests stable across two checkout paths, inputs hashed as esbuild read them. + +import { createHash } from "node:crypto"; +import { mkdir, readFile, realpath, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + type BuildOptions, + type BuiltFacet, + buildFacet, + discoverPlugins, + type Plugin, +} from "../../../src/next-node/plugins/index.ts"; +import { KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +const NODE: BuildOptions = { hostPackages: ["@earendil-works/chord/next"], platform: "node" }; +const BROWSER: BuildOptions = { + hostPackages: ["@earendil-works/chord/next", "react/jsx-runtime"], + platform: "browser", + publicPath: "/assets/", +}; + +/** The plugins under `root`, and the one named `name`. */ +async function discovered(root: string, name: string): Promise<{ plugins: readonly Plugin[]; plugin: Plugin }> { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + return { plugins, plugin: plugins.find((each) => each.name === name)! }; +} + +/** `built`'s script, failing the test with its errors when it failed. */ +function script(built: BuiltFacet): Extract { + if (built.kind === "failed") throw new Error(`build failed: ${built.errors.join("\n")}`); + return built; +} + +/** The errors of a build that must fail. */ +function errors(built: BuiltFacet): readonly string[] { + if (built.kind !== "failed") throw new Error("the build succeeded"); + return built.errors; +} + +/** The sources of `built`'s inline source map. */ +function sourcesOf(built: Extract): string[] { + const inline = /sourceMappingURL=data:application\/json;base64,(\S+)/.exec(built.source)![1]!; + return JSON.parse(Buffer.from(inline, "base64").toString()).sources; +} + +describe("building a facet: the import table", () => { + it("inlines the plugin's own files and another plugin's file (copied); keeps host packages external", async () => { + const root = await tempRoot(["contracts", "notes"]); + const { plugins, plugin } = await discovered(root, "@test/notes"); + const built = script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + const real = await realpath(root); + expect(built).toMatchObject({ plugin: "@test/notes", facet: "worker", assets: [] }); + expect(built.source.startsWith("(function (require, module, exports) {")).toBe(true); + expect(built.source).toContain('require("@earendil-works/chord/next")'); + expect(built.copied).toEqual([join(real, "contracts", "contract.ts")]); + expect(built.inputs.map(({ file }) => file)).toEqual([ + join(real, "contracts", "contract.ts"), + join(real, "notes", "label.ts"), + join(real, "notes", "worker.ts"), + ]); + // Relative to the plugin (a file whose code minified away maps nothing and is left out). + expect(sourcesOf(built)).toEqual(expect.arrayContaining(["worker.ts", "../contracts/contract.ts"])); + for (const source of sourcesOf(built)) expect(source.startsWith("/")).toBe(false); + // Positions only: no copy of the sources' text in the script (Opus N3). + const inline = /sourceMappingURL=data:application\/json;base64,(\S+)/.exec(built.source)![1]!; + expect(JSON.parse(Buffer.from(inline, "base64").toString())).not.toHaveProperty("sourcesContent"); + }); + + it("hashes each input as esbuild read it; the digest is the source's hash", async () => { + const root = await tempRoot(["contracts", "notes"]); + const { plugins, plugin } = await discovered(root, "@test/notes"); + const built = script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + for (const { file, hash } of built.inputs) { + expect(hash).toBe( + createHash("sha256") + .update(await readFile(file)) + .digest("hex"), + ); + } + expect(built.digest).toBe(createHash("sha256").update(built.source).digest("hex")); + }); + + it("gives the same script and digest from two checkout paths (relative source maps)", async () => { + const first = await tempRoot(["contracts", "notes"]); + const second = await tempRoot(["contracts", "notes"]); + const builds = await Promise.all( + [first, second].map(async (root) => { + const { plugins, plugin } = await discovered(root, "@test/notes"); + return script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + }), + ); + expect(builds[0]!.digest).toBe(builds[1]!.digest); + expect(builds[0]!.source).toBe(builds[1]!.source); + }); + + it("inlines a package of its dependencies; keeps chord.external packages and Node builtins external on Node", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "deps", + "@test/deps", + { + "worker.ts": [ + 'import { pad } from "pad-dep";', + 'import native from "native-dep";', + 'import { join } from "node:path";', + 'import { readFileSync } from "fs";', + 'import { pluginFile } from "@earendil-works/chord/next/plugins";', + "export default { setup: () => [pad, native, join, readFileSync, pluginFile] };", + ].join("\n"), + }, + { dependencies: { "pad-dep": "1.0.0" }, chord: { external: ["native-dep"] } }, + ); + await writePackage(dir, "pad-dep", { "index.js": 'exports.pad = (s) => "pad:" + s;' }); + await writePackage(dir, "native-dep", { "index.js": "module.exports = 1;" }); + const { plugin } = await discovered(root, "@test/deps"); + const built = script(await buildFacet(plugin, "worker", NODE, context)); + expect(built.source).toContain('"pad:"'); + for (const external of ["native-dep", "node:path", "fs", "@earendil-works/chord/next/plugins"]) { + expect(built.source).toContain(`require("${external}")`); + } + // Packages are no inputs: the watcher ignores node_modules. + expect(built.inputs.map(({ file }) => file)).toEqual([join(await realpath(dir), "worker.ts")]); + }); + + it("refuses a chord.external package and a Node builtin in a browser build", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "web", + "@test/web", + { + "web.ts": + 'import native from "native-dep";\nimport { join } from "node:path";\nexport default { setup: () => [native, join] };', + }, + { chord: { external: ["native-dep"] } }, + ); + await writePackage(dir, "native-dep", { "index.js": "module.exports = 1;" }); + const { plugin } = await discovered(root, "@test/web"); + expect(errors(await buildFacet(plugin, "web", BROWSER, context))).toEqual([ + 'web.ts:1:20: "native-dep": chord.external packages are for Node builds', + 'web.ts:2:22: "node:path": a Node builtin in a browser build', + ]); + }); + + it("collects a browser build's CSS into one stylesheet and its assets into content-hashed files", async () => { + const root = await tempRoot(); + await writePlugin(root, "web", "@test/web", { + "web.tsx": [ + 'import "./notes.css";', + 'import iconUrl from "./assets/icon.svg?url";', + "const Icon = () => ;", + "export default { setup: () => Icon };", + ].join("\n"), + "notes.css": ".notes { color: red; }", + "assets/icon.svg": "", + }); + const { plugin } = await discovered(root, "@test/web"); + const built = script(await buildFacet(plugin, "web", BROWSER, context)); + expect(built.css?.name).toMatch(/^web-[A-Z0-9]+\.css$/); + expect(built.css?.text).toContain(".notes{color:red}"); + expect(built.assets.map(({ name }) => name)).toEqual([expect.stringMatching(/^icon-[A-Z0-9]+\.svg$/)]); + expect(new TextDecoder().decode(built.assets[0]!.bytes)).toBe(""); + expect(built.source).toContain(`"/assets/${built.assets[0]!.name}"`); + expect(built.source).toContain('require("react/jsx-runtime")'); + }); + + it("scopes a CSS module's class names to its file, the import giving the mapping (slice 11 §4b)", async () => { + const root = await tempRoot(); + await writePlugin(root, "web", "@test/web", { + "web.ts": 'import styles from "./notes.module.css";\nexport default { setup: () => styles.card };', + "notes.module.css": ".card { color: var(--accent); }", + }); + const { plugin } = await discovered(root, "@test/web"); + const built = script(await buildFacet(plugin, "web", BROWSER, context)); + const scoped = /\.(chord_[0-9a-f]{8}_notes_card)\{color:var\(--accent\)\}/.exec(built.css?.text ?? ""); + expect(scoped).not.toBeNull(); + expect(built.source).toContain(`card:"${scoped![1]}"`); + // Another plugin's `notes.module.css` gets other names. + await writePlugin(root, "other", "@test/other", { + "web.ts": 'import styles from "./notes.module.css";\nexport default { setup: () => styles.card };', + "notes.module.css": ".card { color: red; }", + }); + const other = script(await buildFacet((await discovered(root, "@test/other")).plugin, "web", BROWSER, context)); + expect(other.css?.text).toMatch(/\.chord_[0-9a-f]{8}_notes_card\{/); + expect(other.css?.text).not.toContain(scoped![1]); + }); + + it("gives a CSS module's url() assets, its @import, composes and escaped class names (Opus L1)", async () => { + const root = await tempRoot(); + await writePlugin(root, "web", "@test/web", { + "web.ts": 'import styles from "./notes.module.css";\nexport default { setup: () => styles };', + "notes.module.css": [ + '@import "./plain.css";', + ".base { color: red; }", + ".card { composes: base; background: url(./bg.png); }", + ".café { color: blue; }", + ].join("\n"), + "plain.css": ".plain { color: green; }", + "bg.png": "PNG", + }); + const { plugin } = await discovered(root, "@test/web"); + const built = script(await buildFacet(plugin, "web", BROWSER, context)); + const css = built.css?.text ?? ""; + const card = /\.(chord_[0-9a-f]{8}_notes_card)\{/.exec(css)![1]!; + const base = /\.(chord_[0-9a-f]{8}_notes_base)\{/.exec(css)![1]!; + // composes: the class names both; the escaped name maps from the real one. + expect(built.source).toContain(`card:"${base} ${card}"`); + expect(built.source).toMatch(/caf\\u00E9:"chord_[0-9a-f]{8}_notes_caf\\xE9"/); + expect(css).toContain(".plain{color:green}"); + expect(built.assets.map(({ name }) => name)).toEqual([expect.stringMatching(/^bg-[A-Z0-9]+\.png$/)]); + expect(css).toContain(`url("/assets/${built.assets[0]!.name}")`); + }); + + it("gives a plain stylesheet's url() an asset", async () => { + const root = await tempRoot(); + await writePlugin(root, "web", "@test/web", { + "web.ts": 'import "./a.css";\nexport default { setup: () => {} };', + "a.css": ".x { background: url(./bg.png); }", + "bg.png": "PNG", + }); + const { plugin } = await discovered(root, "@test/web"); + const built = script(await buildFacet(plugin, "web", BROWSER, context)); + expect(built.css?.text).toContain(`url("/assets/${built.assets[0]!.name}")`); + }); + + it("refuses CSS and assets in a Node build", async () => { + const root = await tempRoot(); + await writePlugin(root, "n", "@test/n", { + "worker.ts": 'import "./a.css";\nimport url from "./icon.svg?url";\nexport default { setup: () => url };', + "a.css": "a{}", + "icon.svg": "", + }); + const { plugin } = await discovered(root, "@test/n"); + const found = errors(await buildFacet(plugin, "worker", NODE, context)); + expect(found).toEqual([ + 'worker.ts:1:8: "./a.css": stylesheets and assets are for browser builds', + expect.stringMatching(/^worker\.ts:2:17: "\.\/icon\.svg\?url": assets by URL are for browser builds/), + ]); + }); + + it("refuses import.meta.url, __dirname, __filename, a dynamic import() and top-level await, pointing to pluginFile", async () => { + const root = await tempRoot(); + await writePlugin(root, "bad", "@test/bad", { + "worker.ts": "export const a = import.meta.url;\nexport default { setup: () => [__dirname, __filename] };", + "server.ts": 'export default { setup: () => import("./other.ts") };', + "web.ts": "await Promise.resolve();\nexport default { setup: () => {} };", + "other.ts": "export const x = 1;", + }); + const { plugin } = await discovered(root, "@test/bad"); + const meta = errors(await buildFacet(plugin, "worker", NODE, context)); + expect(meta.length).toBeGreaterThan(0); + for (const error of meta) + expect(error).toMatch(/^worker\.ts:\d+:\d+: import\.meta\.url, __dirname and __filename .*pluginFile/); + // Without `import.meta` in the file: each free `__dirname` and `__filename` at its own line (example 37). + await writeFiles(join(root, "bad"), { + "worker.ts": "// The globals.\nconsole.log(__dirname);\nexport default { setup: () => __filename };", + }); + expect(errors(await buildFacet(plugin, "worker", NODE, context))).toEqual([ + expect.stringMatching(/^worker\.ts:2:13: import\.meta\.url, __dirname and __filename .*pluginFile/), + expect.stringMatching(/^worker\.ts:3:\d+: import\.meta\.url, __dirname and __filename .*pluginFile/), + ]); + expect(errors(await buildFacet(plugin, "server", NODE, context))).toEqual([ + expect.stringMatching( + /^server\.ts:1:\d+: import\("\.\/other\.ts"\): a plugin script has no dynamic import .*pluginFile/, + ), + ]); + expect(errors(await buildFacet(plugin, "web", NODE, context))).toEqual([ + "web.ts:1:1: a plugin script has no top-level await (a facet declares synchronously)", + ]); + }); + + it("refuses a path leaving the root, a file in no plugin, a package not in its dependencies, a package in a copied file", async () => { + const root = await tempRoot(["contracts"]); + const away = await tempRoot(); + await writeFiles(away, { "far.ts": "export const far = 1;" }); + await writePlugin(root, "out", "@test/out", { + "worker.ts": [ + 'import { base } from "../base.ts";', + `import { far } from ${JSON.stringify(join(away, "far.ts"))};`, + 'import x from "left-out";', + "export default { setup: () => [base, far, x] };", + ].join("\n"), + }); + await writeFiles(root, { "base.ts": "export const base = 1;" }); + await writePlugin(root, "copier", "@test/copier", { + "worker.ts": 'import { y } from "../shared/shared.ts";\nexport default { setup: () => y };', + }); + await writePlugin(root, "shared", "@test/shared", { + "shared.ts": 'import y from "some-package";\nexport { y };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const out = plugins.find(({ name }) => name === "@test/out")!; + expect(errors(await buildFacet(out, "worker", { ...NODE, plugins }, context))).toEqual([ + 'worker.ts:1:22: "../base.ts": not a file of a plugin in this root', + expect.stringMatching( + /^worker\.ts:2:21: ".*far\.ts": outside the plugin's root \(base code is reached through a host package, other repositories' plugins through dependencies\)$/, + ), + 'worker.ts:3:15: "left-out": neither a host package nor in the dependencies of plugin "@test/out"', + ]); + const copier = plugins.find(({ name }) => name === "@test/copier")!; + expect(errors(await buildFacet(copier, "worker", { ...NODE, plugins }, context))).toEqual([ + '../shared/shared.ts:1:15: "some-package": a file copied from plugin "@test/shared" may import only host packages and files of its own plugin', + ]); + }); + + it("inlines another plugin's files of the same root by relative path; refuses one of another root and names by plugin name", async () => { + const first = await tempRoot(); + const second = await tempRoot(); + await writePlugin(first, "contracts", "@test/contracts", { + "contract.ts": 'export const name = "contract";', + "other.ts": 'import { name } from "./contract.ts";\nexport const other = name;', + }); + await writePlugin(second, "elsewhere", "@test/elsewhere", { "contract.ts": 'export const name = "far";' }); + await writePlugin(first, "user", "@test/user", { + "worker.ts": [ + 'import { name } from "../contracts/contract.ts";', + 'import { other } from "../contracts/other.ts";', + "export default { setup: () => [name, other] };", + ].join("\n"), + }); + await writePlugin(first, "far", "@test/far", { + "worker.ts": `import { name } from ${JSON.stringify(join(second, "elsewhere", "contract.ts"))};\nexport default { setup: () => name };`, + }); + await writePlugin(first, "named", "@test/named", { + "worker.ts": 'import { name } from "@test/contracts/contract";\nexport default { setup: () => name };', + }); + const { plugins } = await discoverPlugins([first, second], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, plugins }, context); + const real = await realpath(first); + expect(script(await build("@test/user")).copied).toEqual([ + join(real, "contracts", "contract.ts"), + join(real, "contracts", "other.ts"), + ]); + expect(errors(await build("@test/far"))).toEqual([ + expect.stringMatching( + /^worker\.ts:1:22: ".*": outside the plugin's root \(base code is reached through a host package/, + ), + ]); + expect(errors(await build("@test/named"))).toEqual([ + 'worker.ts:1:22: "@test/contracts/contract": neither a host package nor in the dependencies of plugin "@test/named"', + ]); + }); + + it("returns a syntax error and a missing facet as failed results, never a throw", async () => { + const root = await tempRoot(); + await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); + const { plugin } = await discovered(root, "@test/syntax"); + expect(errors(await buildFacet(plugin, "worker", NODE, context))).toEqual([ + expect.stringMatching(/^worker\.ts:1:\d+: Expected /), + ]); + expect(await buildFacet(plugin, "server", NODE, context)).toEqual({ + kind: "failed", + plugin: "@test/syntax", + facet: "server", + errors: ['plugin "@test/syntax" has no server facet'], + }); + }); +}); + +/** The app's API of slice 12's tests: two versions, a shared module and per-kind modules. */ +const API = { name: "@acme/app", versions: ["v3", "v2"] } as const; + +/** Each kind's host packages with the API: v3's shared and kind modules, and v2. */ +function apiKind(kind: string): BuildOptions { + return { + hostPackages: ["@acme/app/v3", `@acme/app/v3/${kind}`, "@acme/app/v2"], + platform: kind === "web" ? "browser" : "node", + api: API, + kinds: KINDS, + }; +} + +describe("building a facet: the app's API (slice 12 §1)", () => { + it("keeps a listed version and the kind's module external, and the plugins component's pluginFile", async () => { + const root = await tempRoot(); + await writePlugin(root, "good", "@test/good", { + "worker.ts": [ + 'import { defineFacet } from "@acme/app/v3";', + 'import { Tools } from "@acme/app/v3/worker";', + 'import { pluginFile } from "@earendil-works/chord/next/plugins";', + "export default defineFacet(() => [Tools, pluginFile]);", + ].join("\n"), + }); + const { plugin } = await discovered(root, "@test/good"); + const built = script(await buildFacet(plugin, "worker", apiKind("worker"), context)); + expect(built.source).toContain('require("@acme/app/v3/worker")'); + }); + + it("refuses an unlisted version, no version, another kind's module and Chord itself", async () => { + const root = await tempRoot(); + await writePlugin(root, "bad", "@test/bad", { + "worker.ts": [ + 'import { a } from "@acme/app/v1";', + 'import { b } from "@acme/app";', + 'import { c } from "@acme/app/v3/web";', + 'import { d } from "@earendil-works/chord/next";', + "export default { setup: () => [a, b, c, d] };", + ].join("\n"), + }); + const { plugin } = await discovered(root, "@test/bad"); + // Chord is refused even when the app lists it as a host package. + const options = { + ...apiKind("worker"), + hostPackages: [...apiKind("worker").hostPackages, "@earendil-works/chord/next"], + }; + expect(errors(await buildFacet(plugin, "worker", options, context))).toEqual([ + 'worker.ts:1:19: "@acme/app/v1": @acme/app/v1 is not available: this app provides v3, v2', + 'worker.ts:2:19: "@acme/app": import a version of the app\'s API (this app provides v3, v2)', + 'worker.ts:3:19: "@acme/app/v3/web": not available to worker facets', + 'worker.ts:4:19: "@earendil-works/chord/next": plugins import Chord through the app\'s API ("@acme/app/v3"), never directly', + ]); + }); + + it("refuses two versions in one facet, naming both with their importers", async () => { + const root = await tempRoot(); + await writePlugin(root, "mixed", "@test/mixed", { + "worker.ts": + 'import { a } from "@acme/app/v3";\nimport { b } from "./old.ts";\nexport default { setup: () => [a, b] };', + "old.ts": 'export { b } from "@acme/app/v2";', + }); + const { plugin } = await discovered(root, "@test/mixed"); + expect(errors(await buildFacet(plugin, "worker", apiKind("worker"), context))).toEqual([ + "one facet imports one version of @acme/app: this one imports @acme/app/v2 (old.ts) and @acme/app/v3 (worker.ts)", + ]); + }); + + it("names the app's API in the refusal of a path leaving the root", async () => { + const root = await tempRoot(); + const away = await tempRoot(); + await writeFiles(away, { "base.ts": "export const base = 1;" }); + await writePlugin(root, "out", "@test/out", { + "worker.ts": `import { base } from ${JSON.stringify(join(away, "base.ts"))};\nexport default { setup: () => base };`, + }); + const { plugin } = await discovered(root, "@test/out"); + expect(errors(await buildFacet(plugin, "worker", apiKind("worker"), context))).toEqual([ + expect.stringMatching(/base code is reached through the app's API \("@acme\/app\/v3"\)/), + ]); + }); +}); + +describe("building a facet: per-kind files (slice 12 §3)", () => { + it("allows a file of the facet's own kind; refuses one of another kind, naming the chain that reached it", async () => { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { + "contract.ts": 'export const shared = "shared";', + "contract.worker.ts": 'export const tools = "tools";', + "contract.web.tsx": 'export const panel = "panel";', + "view.ts": 'export { panel } from "./contract.web.tsx";', + "worker.ts": [ + 'import { shared } from "./contract.ts";', + 'import { tools } from "./contract.worker.ts";', + "export default { setup: () => [shared, tools] };", + ].join("\n"), + "server.ts": 'import { panel } from "./view.ts";\nexport default { setup: () => panel };', + }); + const { plugins, plugin } = await discovered(root, "@test/notes"); + script(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS, plugins }, context)); + expect(errors(await buildFacet(plugin, "server", { ...NODE, kinds: KINDS, plugins }, context))).toEqual([ + 'view.ts:1:23: "./contract.web.tsx": a web file, imported into a server facet (server.ts → view.ts → contract.web.tsx)', + ]); + }); + + it("checks another plugin's copied files; never a package's file names (pdfjs's pdf.worker.mjs)", async () => { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { "contract.web.tsx": 'export const panel = "panel";' }); + const user = await writePlugin( + root, + "user", + "@test/user", + { + "server.ts": 'import { panel } from "../notes/contract.web.tsx";\nexport default { setup: () => panel };', + "worker.ts": + 'import { list } from "@bob/notes/web";\nimport pdf from "pdfjs-like";\nexport default { setup: () => [list, pdf] };', + }, + { dependencies: { "@bob/notes": "1.0.0", "pdfjs-like": "1.0.0" } }, + ); + await writePackage( + user, + "@bob/notes", + { "contract.ts": "export const list = 1;", "contract.web.tsx": "export const list = 2;" }, + { exports: { ".": "./contract.ts", "./web": "./contract.web.tsx" } }, + ); + await writePackage( + user, + "pdfjs-like", + { + "index.mjs": 'export { default } from "./build/pdf.worker.mjs";', + "build/pdf.worker.mjs": "export default 1;", + }, + { type: "module", main: "index.mjs" }, + ); + const { plugins, plugin } = await discovered(root, "@test/user"); + const options = { ...NODE, kinds: KINDS, plugins }; + expect(errors(await buildFacet(plugin, "server", options, context))).toEqual([ + 'server.ts:1:23: "../notes/contract.web.tsx": a web file, imported into a server facet (server.ts → ../notes/contract.web.tsx)', + ]); + script(await buildFacet(plugin, "worker", options, context)); + }); + + it("refuses another kind's facet file and a per-kind name of any case (Opus L1, M2)", async () => { + const root = await tempRoot(); + await writePlugin(root, "f", "@test/f", { + "worker.ts": + 'import { a } from "./web.tsx";\nimport { b } from "./contract.Web.ts";\nexport default { setup: () => [a, b] };', + "web.tsx": "export const a = 1;", + "contract.Web.ts": "export const b = 2;", + }); + const { plugins, plugin } = await discovered(root, "@test/f"); + expect(errors(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS, plugins }, context))).toEqual([ + 'worker.ts:1:19: "./web.tsx": a web file, imported into a worker facet (worker.ts → web.tsx)', + 'worker.ts:2:19: "./contract.Web.ts": a web file, imported into a worker facet (worker.ts → contract.Web.ts)', + ]); + }); + + it("names the chain of an import cycle without hanging (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "cycle", "@test/cycle", { + "worker.ts": 'import { value } from "./a.ts";\nexport default { setup: () => value };', + "a.ts": 'import "./worker.ts";\nexport { value } from "./bridge.ts";', + "bridge.ts": 'export { value } from "./bad.web.ts";', + "bad.web.ts": "export const value = 1;", + }); + const { plugins, plugin } = await discovered(root, "@test/cycle"); + expect(errors(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS, plugins }, context))).toEqual([ + 'bridge.ts:1:23: "./bad.web.ts": a web file, imported into a worker facet (worker.ts → a.ts → bridge.ts → bad.web.ts)', + ]); + }); +}); + +describe("building a facet: roots under node_modules and plugin packages (slice 12 fix round 2: Opus N1, N2)", () => { + it("builds a plugin of a root that lies under a node_modules (an app installed from npm with its plugins)", async () => { + const app = await tempRoot(); + const root = join(app, "node_modules", "@acme", "app", "plugins"); + await writePlugin(root, "notes", "@acme/notes", { + "worker.ts": + 'import { x } from "./helper.ts";\nimport { c } from "../contracts/contract.ts";\nexport default { setup: () => [x, c] };', + "helper.ts": "export const x = 1;", + }); + await writePlugin(root, "contracts", "@acme/contracts", { "contract.ts": "export const c = 1;" }); + const { plugins, plugin } = await discovered(root, "@acme/notes"); + expect(script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)).copied).toEqual([ + join(await realpath(root), "contracts", "contract.ts"), + ]); + }); + + it("checks per-kind files in a package that is plugin code (a chord field), not in a library", async () => { + const root = await tempRoot(); + const user = await writePlugin( + root, + "user", + "@test/user", + { + "worker.ts": + 'import { panel } from "@bob/plugins/notes/contract.web.tsx";\nexport default { setup: () => panel };', + "server.ts": 'import { a } from "@bob/plugins/notes/web.tsx";\nexport default { setup: () => a };', + }, + { dependencies: { "@bob/plugins": "1.0.0" } }, + ); + await writeFiles(join(user, "node_modules", "@bob", "plugins"), { + "package.json": JSON.stringify({ name: "@bob/plugins", chord: { plugins: ["notes"] } }), + "notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "notes/contract.web.tsx": 'export const panel = "panel";', + "notes/web.tsx": "export const a = 1;", + }); + const { plugins, plugin } = await discovered(root, "@test/user"); + const options = { ...NODE, kinds: KINDS, plugins }; + expect(errors(await buildFacet(plugin, "worker", options, context))).toEqual([ + 'worker.ts:1:23: "@bob/plugins/notes/contract.web.tsx": a web file, imported into a worker facet (worker.ts → node_modules/@bob/plugins/notes/contract.web.tsx)', + ]); + expect(errors(await buildFacet(plugin, "server", options, context))).toEqual([ + 'server.ts:1:19: "@bob/plugins/notes/web.tsx": a web file, imported into a server facet (server.ts → node_modules/@bob/plugins/notes/web.tsx)', + ]); + }); +}); + +describe("building a facet: libraries inside plugin packages (slice 12 fix round 3: Opus R2)", () => { + it("leaves a library unchecked by kind, also one nested in a plugin package's own node_modules", async () => { + const root = await tempRoot(); + const user = await writePlugin( + root, + "p", + "@test/p", + { "server.ts": 'import x from "@bob/plugins/notes/contract.ts";\nexport default { setup: () => x };' }, + { dependencies: { "@bob/plugins": "1.0.0" } }, + ); + await writeFiles(join(user, "node_modules", "@bob", "plugins"), { + "package.json": JSON.stringify({ name: "@bob/plugins", chord: { plugins: ["./notes"] } }), + "notes/package.json": JSON.stringify({ name: "@bob/notes", dependencies: { "pdfjs-like": "1.0.0" } }), + "notes/contract.ts": 'import pdf from "pdfjs-like";\nexport default pdf;', + "notes/node_modules/pdfjs-like/package.json": JSON.stringify({ + name: "pdfjs-like", + type: "module", + main: "index.mjs", + }), + "notes/node_modules/pdfjs-like/index.mjs": 'export { default } from "./build/pdf.worker.mjs";', + "notes/node_modules/pdfjs-like/build/pdf.worker.mjs": "export default 1;", + }); + const { plugins, plugin } = await discovered(root, "@test/p"); + script(await buildFacet(plugin, "server", { ...NODE, kinds: KINDS, plugins }, context)); + }); +}); + +describe("building a facet: what lies outside the plugin (slice 12 fix round 1: Opus M1, Sol)", () => { + it("refuses a package file whose real path is outside the plugin's node_modules", async () => { + const app = await tempRoot(); + await writeFiles(app, { + "src/core.ts": "export const SECRET = 'base code';", + "src/package.json": '{ "name": "base", "main": "core.ts" }', + }); + const root = join(app, "plugins"); + const a = await writePlugin( + root, + "a", + "@test/a", + { "worker.ts": 'import { SECRET } from "base";\nexport default { setup: () => SECRET };' }, + { dependencies: { base: "file:../../src" } }, + ); + await mkdir(join(a, "node_modules"), { recursive: true }); + // npm's `file:` dependency: a link out of node_modules. + await symlink(join(app, "src"), join(a, "node_modules", "base"), "dir"); + const b = await writePlugin( + root, + "b", + "@test/b", + { "worker.ts": 'import { S } from "dep";\nexport default { setup: () => S };' }, + { dependencies: { dep: "1.0.0" } }, + ); + await writePackage(b, "dep", { "index.js": 'export { SECRET as S } from "../../../../src/core.ts";' }); + const c = await writePlugin( + root, + "c", + "@test/c", + { "worker.ts": 'import { S } from "dep";\nexport default { setup: () => S };' }, + { dependencies: { dep: "1.0.0" } }, + ); + await writePackage(c, "dep", { "package.json": '{ "name": "dep", "main": "index.js" }' }); + // A link inside the package pointing out of it. + await symlink(join(app, "src", "core.ts"), join(c, "node_modules", "dep", "index.js")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, plugins }, context); + const outside = /: resolves outside the plugin's node_modules \(.*src\/core\.ts\)$/; + expect(errors(await build("@test/a"))).toEqual([expect.stringMatching(outside)]); + expect(errors(await build("@test/b"))).toEqual([expect.stringMatching(outside)]); + expect(errors(await build("@test/c"))).toEqual([expect.stringMatching(outside)]); + }); + + it("refuses a facet file linked out of its plugin, or to a file of another kind (Sol)", async () => { + const root = await tempRoot(); + const away = await tempRoot(); + await writeFiles(away, { "facet.ts": "export default { setup: () => {} };" }); + const out = await writePlugin(root, "out", "@test/out", {}); + await symlink(join(away, "facet.ts"), join(out, "worker.ts")); + const kind = await writePlugin(root, "kind", "@test/kind", { "web.tsx": "export default { setup: () => {} };" }); + await symlink(join(kind, "web.tsx"), join(kind, "worker.ts")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "worker", { ...NODE, kinds: KINDS, plugins }, context); + expect(errors(await build("@test/out"))).toEqual([ + expect.stringMatching(/^worker\.ts: the facet file links out of its plugin/), + ]); + expect(errors(await build("@test/kind"))).toEqual([ + "worker.ts: links to a web file (web.tsx), not a worker facet", + ]); + }); + + it("refuses a Node builtin in a copied file (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "other", "@test/other", { + "contract.ts": 'import { basename } from "node:path";\nexport const value = basename("x/y");', + }); + await writePlugin(root, "user", "@test/user", { + "worker.ts": 'import { value } from "../other/contract.ts";\nexport default { setup: () => value };', + }); + const { plugins, plugin } = await discovered(root, "@test/user"); + expect(errors(await buildFacet(plugin, "worker", { ...NODE, plugins }, context))).toEqual([ + '../other/contract.ts:1:26: "node:path": a file copied from plugin "@test/other" may import only host packages and files of its own plugin', + ]); + }); +}); + +describe("building a facet: packages through their exports (slice 12 §2)", () => { + it("inlines a dependency's shared and per-kind contract files as its exports map them", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "stats", + "@test/stats", + { + "worker.ts": 'import { Notes } from "@bob/notes";\nexport default { setup: () => Notes };', + "web.tsx": 'import { Panel } from "@bob/notes/web";\nexport default { setup: () => Panel };', + }, + { dependencies: { "@bob/notes": "1.0.0" } }, + ); + await writePackage( + dir, + "@bob/notes", + { + "contract.ts": 'export const Notes = "notes v1";', + "contract.web.tsx": 'export const Panel = "panel v1";', + }, + { exports: { ".": "./contract.ts", "./web": "./contract.web.tsx" }, type: "module" }, + ); + const { plugin } = await discovered(root, "@test/stats"); + const worker = script(await buildFacet(plugin, "worker", { ...NODE, kinds: KINDS }, context)); + expect(worker.source).toContain("notes v1"); + const web = script(await buildFacet(plugin, "web", { ...BROWSER, kinds: KINDS }, context)); + expect(web.source).toContain("panel v1"); + // Packages are not the watcher's inputs. + expect(worker.inputs.map(({ file }) => file)).toEqual([join(await realpath(dir), "worker.ts")]); + }); +}); + +describe("building a facet: single-file plugins (slice 12 §4)", () => { + it("allows host packages only, refusing other files, packages and builtins with a pointer to a directory", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.server.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});', + "bad.server.ts": [ + 'import { a } from "./hello.server.ts";', + 'import b from "some-package";', + 'import { join } from "node:path";', + "export default { setup: () => [a, b, join] };", + ].join("\n"), + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const build = (name: string) => + buildFacet(plugins.find((each) => each.name === name)!, "server", { ...NODE, plugins }, context); + expect(sourcesOf(script(await build("hello")))).toEqual(["hello.server.ts"]); + const hint = + 'a single-file plugin imports only host packages (the app\'s API); make it a directory "bad/" with a package.json'; + expect(errors(await build("bad"))).toEqual([ + `bad.server.ts:1:19: "./hello.server.ts": ${hint}`, + `bad.server.ts:2:15: "some-package": ${hint}`, + `bad.server.ts:3:22: "node:path": ${hint}`, + ]); + }); +}); diff --git a/packages/chord/test/next-node/plugins/discover.test.ts b/packages/chord/test/next-node/plugins/discover.test.ts new file mode 100644 index 000000000..2fdfd55d3 --- /dev/null +++ b/packages/chord/test/next-node/plugins/discover.test.ts @@ -0,0 +1,272 @@ +// Tests of discovery (slice 7c §1, slice 12 §1, §2, §4): names from package.json, the first root winning a duplicate, +// directories reported (no or an invalid package.json, an invalid name, two files of one kind), contracts-only plugins, +// symlinks, required roots, checkouts of several plugins, packages' chord.plugins manifests, single-file plugins. + +import { mkdir, realpath, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { discoverPlugins } from "../../../src/next-node/plugins/index.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +describe("discovery", () => { + it("names a plugin by its package.json, lists its facet files by kind, its dependencies and externals", async () => { + const root = await tempRoot(["contracts", "notes"]); + const dir = await writePlugin( + root, + "deps-dir", + "@test/deps", + { "worker.ts": "", "store.ts": "" }, + { + dependencies: { marked: "1.0.0" }, + chord: { external: ["better-sqlite3"] }, + }, + ); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + const real = await realpath(root); + const of = { root: real, required: false, layout: "directory" } as const; + expect(plugins).toEqual([ + { name: "@test/contracts", dir: join(real, "contracts"), ...of, facets: {}, dependencies: [], external: [] }, + { + name: "@test/deps", + dir: await realpath(dir), + ...of, + facets: { worker: join(real, "deps-dir", "worker.ts") }, + dependencies: ["marked"], + external: ["better-sqlite3"], + }, + { + name: "@test/notes", + dir: join(real, "notes"), + ...of, + facets: { server: join(real, "notes", "server.ts"), worker: join(real, "notes", "worker.ts") }, + dependencies: [], + external: [], + }, + ]); + }); + + it("reports a directory without package.json, with an invalid one, an invalid name or two files of a kind", async () => { + const root = await tempRoot(); + await mkdir(join(root, "bare")); + await writeFiles(join(root, "broken"), { "package.json": "{ not json" }); + await writePlugin(root, "invalid", "Not A Name", {}); + await writePlugin(root, "two", "@test/two", { "worker.ts": "", "worker.js": "" }); + await writePlugin(root, "good", "@test/good", { "web.tsx": "" }); + // Not plugins and not reported: hidden directories and a root's own packages. + await mkdir(join(root, ".git")); + await mkdir(join(root, "node_modules")); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@test/good"]); + const real = await realpath(root); + expect(problems.map(({ path, error }) => ({ path, error }))).toEqual([ + { path: join(real, "bare"), error: "no package.json" }, + { path: join(real, "broken"), error: expect.stringMatching(/^package\.json: /) }, + { path: join(real, "invalid"), error: 'package.json: "Not A Name" is not a valid plugin name' }, + { path: join(real, "two"), error: "two worker facets: worker.ts, worker.js" }, + ]); + }); + + it("keeps the first root's plugin of a duplicate name and reports the other; a missing root is empty", async () => { + const first = await tempRoot(); + const second = await tempRoot(); + await writePlugin(first, "a", "@test/same", { "worker.ts": "" }); + await writePlugin(second, "b", "@test/same", { "server.ts": "" }); + const { plugins, problems } = await discoverPlugins( + [first, join(first, "missing"), second], + { kinds: KINDS }, + context, + ); + expect(plugins.map(({ dir }) => dir)).toEqual([join(await realpath(first), "a")]); + expect(problems.map(({ path, error }) => ({ path, error }))).toEqual([ + { + path: join(await realpath(second), "b"), + error: `plugin "@test/same" is in ${join(await realpath(first), "a")} already`, + }, + ]); + }); + + it("follows a symlinked plugin directory to its real path", async () => { + const elsewhere = await tempRoot(); + const root = await tempRoot(); + const dir = await writePlugin(elsewhere, "linked", "@test/linked", { "worker.ts": "" }); + await symlink(dir, join(root, "link")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map((each) => [each.name, each.dir])).toEqual([["@test/linked", await realpath(dir)]]); + expect(plugins[0]!.facets).toEqual({ worker: join(await realpath(dir), "worker.ts") }); + }); +}); + +describe("discovery: roots and checkouts (slice 12)", () => { + it("marks the plugins of a required root and records each plugin's root", async () => { + const app = await tempRoot(); + const user = await tempRoot(); + await writePlugin(app, "core", "@test/core", { "server.ts": "" }); + await writePlugin(user, "extra", "@test/extra", { "server.ts": "" }); + const { plugins } = await discoverPlugins([{ path: app, required: true }, user], { kinds: KINDS }, context); + expect(plugins.map(({ name, root, required }) => [name, root, required])).toEqual([ + ["@test/core", await realpath(app), true], + ["@test/extra", await realpath(user), false], + ]); + }); + + it("lists the plugins of a checkout of several (a directory without package.json), within its root", async () => { + const root = await tempRoot(); + await writePlugin(join(root, "repo"), "notes", "@test/notes", { "worker.ts": "" }); + await writePlugin(join(root, "repo"), "search", "@test/search", { "worker.ts": "" }); + await mkdir(join(root, "repo", "docs")); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(problems).toEqual([]); + expect(plugins.map(({ name, dir, root: of }) => [name, dir, of])).toEqual([ + ["@test/notes", join(real, "repo", "notes"), real], + ["@test/search", join(real, "repo", "search"), real], + ]); + }); +}); + +describe("discovery: packages and their chord.plugins manifests (slice 12 §2)", () => { + it("lists the folders chord.plugins names: paths, globs and exclusions", async () => { + const root = await tempRoot(); + await writeFiles(join(root, "bob"), { + "package.json": JSON.stringify({ + name: "@bob/plugins", + chord: { plugins: ["./notes/", "extra/*", "!extra/old", "deep/**/p?"] }, + }), + "server.ts": "", + }); + await writePlugin(join(root, "bob"), "notes", "@bob/notes", { "worker.ts": "" }); + await writePlugin(join(root, "bob"), "unlisted", "@bob/unlisted", { "worker.ts": "" }); + await writePlugin(join(root, "bob", "extra"), "search", "@bob/search", { "worker.ts": "" }); + await writePlugin(join(root, "bob", "extra"), "old", "@bob/old", { "worker.ts": "" }); + await writePlugin(join(root, "bob", "deep", "a", "b"), "p1", "@bob/p1", { "worker.ts": "" }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + const real = await realpath(root); + expect(plugins.map(({ name, dir }) => [name, dir.slice(real.length + 1)])).toEqual([ + ["@bob/notes", "bob/notes"], + ["@bob/p1", "bob/deep/a/b/p1"], + ["@bob/search", "bob/extra/search"], + ]); + }); + + it("lists the package itself as `.`, and reports a listed folder inside another listed one (Opus N5)", async () => { + const root = await tempRoot(); + await writeFiles(join(root, "one"), { + "package.json": JSON.stringify({ name: "@bob/one", chord: { plugins: ["."] } }), + "worker.ts": "", + }); + await writeFiles(join(root, "two"), { + "package.json": JSON.stringify({ name: "@bob/two", chord: { plugins: [".", "./sub"] } }), + "sub/package.json": JSON.stringify({ name: "@bob/sub" }), + }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@bob/one"]); + expect(problems.map(({ error }) => error)).toEqual([ + 'package.json: chord.plugins lists "sub" inside the listed "."', + ]); + }); + + it("reports a manifest that is no list, lists nothing, leaves the package or names no directory", async () => { + const root = await tempRoot(); + const manifest = (plugins: unknown) => JSON.stringify({ name: "@bob/x", chord: { plugins } }); + await writeFiles(root, { + "a/package.json": manifest({ "@bob/notes": "github:bob/plugins#v1:notes" }), + "b/package.json": manifest(["nothing/*"]), + "c/package.json": manifest(["../a"]), + "d/package.json": manifest(["./file.txt"]), + "d/file.txt": "", + }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(plugins).toEqual([]); + expect(problems.map(({ path, error }) => [path.slice(real.length + 1), error])).toEqual([ + ["a", "package.json: chord.plugins is not a list of plugin folders"], + ["b", "package.json: chord.plugins lists no plugin folder"], + ["c", 'package.json: chord.plugins "../a" leaves the package'], + ["d/file.txt", "listed in chord.plugins, not a directory"], + ]); + }); + + it("by convention: a package with facet files at its top is one plugin, else its subfolders with a package.json", async () => { + const root = await tempRoot(); + await writePlugin(root, "single", "@test/single", { "worker.ts": "", "fixtures/package.json": "{}" }); + await writeFiles(join(root, "repo"), { "package.json": JSON.stringify({ name: "@test/repo" }) }); + await writePlugin(join(root, "repo"), "notes", "@test/notes", { "worker.ts": "" }); + await writePlugin(root, "contracts-only", "@test/contracts", { "contract.ts": "" }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + expect(plugins.map(({ name }) => name)).toEqual(["@test/contracts", "@test/notes", "@test/single"]); + }); + + it("reports a required root that does not exist, and records each problem's root", async () => { + const root = await tempRoot(); + await writeFiles(join(root, "broken"), { "package.json": "{ not json" }); + const missing = join(root, "missing"); + const { problems } = await discoverPlugins( + [ + { path: root, required: true }, + { path: missing, required: true }, + ], + { kinds: KINDS }, + context, + ); + expect(problems).toEqual([ + { + path: join(await realpath(root), "broken"), + error: expect.stringMatching(/^package\.json: /), + root: await realpath(root), + required: true, + }, + { path: missing, error: "the required root does not exist", root: missing, required: true }, + ]); + }); +}); + +describe("discovery: single-file plugins (slice 12 §4)", () => { + it("groups .. files at a root's top by name, one facet per file, without package.json", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.server.ts": "", + "hello.web.tsx": "", + "clock.worker.mjs": "", + "notes.md": "", + "helper.ts": "", + "other.thing.ts": "", + }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(problems).toEqual([]); + const of = { dir: real, root: real, required: false, layout: "file", dependencies: [], external: [] }; + expect(plugins).toEqual([ + { name: "clock", ...of, facets: { worker: join(real, "clock.worker.mjs") } }, + { + name: "hello", + ...of, + facets: { server: join(real, "hello.server.ts"), web: join(real, "hello.web.tsx") }, + }, + ]); + }); + + it("reports an invalid name and two files of a kind; a directory plugin of the same name wins", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "Bad.server.ts": "", + "twice.server.ts": "", + "twice.server.js": "", + "notes.server.ts": "", + }); + const dir = await writePlugin(root, "zz-notes", "notes", { "server.ts": "" }); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + const real = await realpath(root); + expect(plugins.map(({ name, layout, dir: at }) => [name, layout, at])).toEqual([ + ["notes", "directory", await realpath(dir)], + ]); + expect(problems.map(({ path, error }) => ({ path, error }))).toEqual([ + { path: join(real, "Bad.server.ts"), error: '"Bad" is not a valid unscoped plugin name' }, + { path: join(real, "twice.server.js"), error: "two server facets: twice.server.js, twice.server.ts" }, + { path: join(real, "notes.server.ts"), error: `plugin "notes" is in ${await realpath(dir)} already` }, + ]); + }); +}); diff --git a/packages/chord/test/next-node/plugins/evaluate.test.ts b/packages/chord/test/next-node/plugins/evaluate.test.ts new file mode 100644 index 000000000..7186475ff --- /dev/null +++ b/packages/chord/test/next-node/plugins/evaluate.test.ts @@ -0,0 +1,275 @@ +// Tests of evaluation (slice 7c §3): host packages as the host's instances (`instanceof ChordError` across a plugin), +// default and named imports of ESM and CommonJS host packages, chord.external packages and builtins from the plugin, +// `pluginFile` and its `..` refusal (and a single-file plugin's, slice 12 §4), an import-time throw, a non-facet export, +// a refused require, a computed import(), the filename. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { ChordError, isChordError } from "../../../src/next/data/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { + type BuiltScript, + buildFacet, + discoverPlugins, + evaluateFacet, + type Plugin, + pluginFile, +} from "../../../src/next-node/plugins/index.ts"; +import { expectCollected } from "../../next/support.ts"; +import * as esmHost from "../fixtures/esm-host.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +const HOSTS = { ...HOST_PACKAGES, "esm-host": esmHost, "cjs-host": { named: "cjs named" } }; + +/** The fixtures' `Files` contract, as the `contracts` fixture declares it. */ +const Files = local("test.files", { version: 1, params: {}, members: { text: str() } }); + +/** Builds `name`'s `kind` facet under `root` for Node and returns the script and the plugin. */ +async function built(root: string, name: string, kind = "worker"): Promise<{ script: BuiltScript; plugin: Plugin }> { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins.find((each) => each.name === name)!; + const result = await buildFacet( + plugin, + kind, + { hostPackages: Object.keys(HOSTS), platform: "node", plugins }, + context, + ); + if (result.kind === "failed") throw new Error(result.errors.join("\n")); + return { script: result, plugin }; +} + +/** A facet with extra members a test reads. */ +type Probe = { readonly setup: () => void; readonly [name: string]: unknown }; + +describe("evaluating a facet", () => { + it("gives host packages as the host's instances: a ChordError crosses the plugin as one", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": [ + 'import { ChordError, defineFacet } from "@earendil-works/chord/next";', + "const facet = defineFacet(() => {});", + 'export default { ...facet, made: new ChordError({ code: "unavailable", message: "m" }), is: (e: unknown) => e instanceof ChordError };', + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.made).toBeInstanceOf(ChordError); + expect(isChordError(facet.made, "unavailable")).toBe(true); + expect((facet.is as (e: unknown) => boolean)(new ChordError({ code: "unavailable", message: "x" }))).toBe(true); + }); + + it("answers default and named imports of an ESM and a CommonJS host package", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": [ + 'import esm, { named as esmNamed } from "esm-host";', + 'import * as esmAll from "esm-host";', + 'import cjs, { named as cjsNamed } from "cjs-host";', + "export default { setup: () => {}, values: [esm, esmNamed, esmAll.named, cjs, cjsNamed] };", + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.values).toEqual(["esm default", "esm named", "esm named", { named: "cjs named" }, "cjs named"]); + }); + + it("loads chord.external packages and Node builtins from the plugin's directory", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "p", + "@test/p", + { + "worker.ts": + 'import ext from "ext-dep";\nimport { basename } from "node:path";\nexport default { setup: () => {}, values: [ext, basename("/a/b")] };', + }, + { chord: { external: ["ext-dep"] } }, + ); + await writePackage(dir, "ext-dep", { "index.js": 'module.exports = "from the plugin";' }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.values).toEqual(["from the plugin", "b"]); + }); + + it("gives each facet pluginFile for its own plugin's directory and refuses .. out of it", async () => { + const root = await tempRoot(["contracts", "files"]); + const { script, plugin } = await built(root, "@test/files"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/files", facet }], context); + expect((await host.get(Files, {}, context)).text.value).toBe("a file of the plugin"); + await host.close(context); + await writePlugin(root, "escape", "@test/escape", { + "worker.ts": + 'import { pluginFile } from "@earendil-works/chord/next/plugins";\nexport default { setup: () => {}, escape: () => pluginFile("../files/assets/data.txt") };', + }); + const escaping = await built(root, "@test/escape"); + const probe = evaluateFacet(escaping.script, { + hostPackages: HOSTS, + plugin: escaping.plugin, + }) as unknown as Probe; + expect(() => (probe.escape as () => string)()).toThrow( + 'pluginFile("../files/assets/data.txt"): outside the directory of plugin "@test/escape"', + ); + // Outside an evaluated facet it has none. + expect(() => pluginFile("x")).toThrow("only a plugin's facet script"); + }); + + it("refuses pluginFile in a single-file plugin: it has no directory of its own (slice 12 §4)", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "lone.worker.ts": + 'import { pluginFile } from "@earendil-works/chord/next/plugins";\nexport default { setup: () => {}, read: () => pluginFile("x.txt") };', + }); + const { script, plugin } = await built(root, "lone"); + const probe = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(() => (probe.read as () => string)()).toThrow( + 'pluginFile("x.txt"): single-file plugin "lone" has no directory of its own', + ); + }); + + it("throws an import-time throw, a non-facet export and a refused require, naming the facet", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'throw new Error("import time");\nexport default { setup: () => {} };', + "server.ts": "export default 42;", + }); + const thrown = await built(root, "@test/p"); + expect(() => evaluateFacet(thrown.script, { hostPackages: HOSTS, plugin: thrown.plugin })).toThrow("import time"); + const notFacet = await built(root, "@test/p", "server"); + expect(() => evaluateFacet(notFacet.script, { hostPackages: HOSTS, plugin: notFacet.plugin })).toThrow( + 'plugin "@test/p" (server): its default export is not a facet (defineFacet)', + ); + const handmade: BuiltScript = { + ...thrown.script, + source: '(function (require, module, exports) { require("nope"); })', + }; + expect(() => evaluateFacet(handmade, { hostPackages: HOSTS, plugin: thrown.plugin })).toThrow( + 'plugin "@test/p" (worker): "nope" is not a host package, a Node builtin or one of its chord.external packages', + ); + }); + + it("runs a computed import() through the facet's require: an unknown one names the facet (Astra)", async () => { + const root = await tempRoot(); + await writePlugin(root, "dynamic", "@test/dynamic", { + "worker.ts": [ + 'import { defineFacet } from "@earendil-works/chord/next";', + "const later = (specifier: string) => import(specifier);", + "export default { ...defineFacet(() => {}), later };", + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/dynamic"); + // The fixture attaches its computed import to the facet. + const { later } = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as { + later: (specifier: string) => Promise<{ readonly named?: string }>; + }; + await expect(later("missing-package")).rejects.toThrow( + 'plugin "@test/dynamic" (worker): "missing-package" is not a host package', + ); + expect((await later("cjs-host")).named).toBe("cjs named"); + }); + + it("runs a script under a stable filename per plugin, kind and digest", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'export default { setup: () => {}, fail: () => { throw new Error("x"); } };', + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + let stack = ""; + try { + (facet.fail as () => void)(); + } catch (error) { + stack = (error as Error).stack!; + } + expect(stack).toContain(`/chord-plugins/@test/p/worker.ts?${script.digest}:`); + }); +}); + +describe("a computed require refuses what the build refuses (slice 12 fix round 1, Sol)", () => { + it("refuses another version of the app's API, Chord itself, and builtins in a single-file plugin", async () => { + const root = await tempRoot(); + const api = { name: "@acme/app", versions: ["v3", "v2"] }; + const modules = { "@acme/app/v3": { value: 3 }, "@acme/app/v2": { value: 2 }, "@earendil-works/chord/next": {} }; + const join = (...parts: string[]) => `[${parts.map((part) => JSON.stringify(part)).join(", ")}].join("/")`; + await writePlugin(root, "mixed", "@test/mixed", { + "worker.ts": [ + 'import { value } from "@acme/app/v3";', + `export default { setup: () => {}, other: () => require(${join("@acme/app", "v2")}), chord: () => require(${join("@earendil-works", "chord", "next")}), same: () => require(${join("@acme/app", "v3")}).value + value };`, + ].join("\n"), + }); + await writeFiles(root, { + "lone.worker.ts": 'export default { setup: () => {}, builtin: () => require(["node", "path"].join(":")) };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const options = { hostPackages: Object.keys(modules), platform: "node", api, kinds: KINDS, plugins } as const; + const evaluated = async (name: string) => { + const plugin = plugins.find((each) => each.name === name)!; + const built = await buildFacet(plugin, "worker", options, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return evaluateFacet(built, { hostPackages: modules, plugin }) as unknown as Probe; + }; + const mixed = await evaluated("@test/mixed"); + expect((mixed.same as () => number)()).toBe(6); + expect(() => (mixed.other as () => unknown)()).toThrow( + '"@acme/app/v2": this script uses @acme/app/v3: one facet uses one version of the API', + ); + expect(() => (mixed.chord as () => unknown)()).toThrow("plugins import Chord through the app's API"); + // Without a static import the first version required pins the facet's (Opus N3). + await writePlugin(root, "computed", "@test/computed", { + "worker.ts": `export default { setup: () => {}, both: () => [require(${join("@acme/app", "v3")}).value, require(${join("@acme/app", "v2")}).value] };`, + }); + const fresh = await discoverPlugins([root], { kinds: KINDS }, context); + const computedPlugin = fresh.plugins.find((each) => each.name === "@test/computed")!; + const computedBuilt = await buildFacet(computedPlugin, "worker", { ...options, plugins: fresh.plugins }, context); + if (computedBuilt.kind === "failed") throw new Error(computedBuilt.errors.join("\n")); + const computed = evaluateFacet(computedBuilt, { + hostPackages: modules, + plugin: computedPlugin, + }) as unknown as Probe; + expect(() => (computed.both as () => unknown)()).toThrow( + '"@acme/app/v2": this script uses @acme/app/v3: one facet uses one version of the API', + ); + const lone = await evaluated("lone"); + expect(() => (lone.builtin as () => unknown)()).toThrow( + '"node:path": a single-file plugin requires only host packages', + ); + }); +}); + +describe("ESM host packages' wrappers", () => { + it("wraps a namespace once, and keeps no namespace nobody else holds", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'import * as ns from "fake-esm";\nexport default { setup: () => {}, ns };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const result = await buildFacet(plugin, "worker", { hostPackages: ["fake-esm"], platform: "node" }, context); + if (result.kind === "failed") throw new Error(result.errors.join("\n")); + // What `Object.prototype.toString` calls a module namespace. + const evaluate = (namespace: object) => + evaluateFacet(result, { hostPackages: { "fake-esm": namespace }, plugin }) as unknown as Probe; + // Wrapping reads each export once: twice evaluated, read once. + let reads = 0; + const namespace = { + [Symbol.toStringTag]: "Module", + get named() { + reads++; + return 1; + }, + }; + expect([ + (evaluate(namespace).ns as { named: number }).named, + (evaluate(namespace).ns as { named: number }).named, + ]).toEqual([1, 1]); + expect(reads).toBe(1); + const ref = (() => { + const dropped = { [Symbol.toStringTag]: "Module", named: 2 }; + expect((evaluate(dropped).ns as { named: number }).named).toBe(2); + return new WeakRef(dropped); + })(); + await expectCollected(ref); + }); +}); diff --git a/packages/chord/test/next-node/plugins/install.test.ts b/packages/chord/test/next-node/plugins/install.test.ts new file mode 100644 index 000000000..d45849d44 --- /dev/null +++ b/packages/chord/test/next-node/plugins/install.test.ts @@ -0,0 +1,302 @@ +// Tests of installing (slice 7c §5, slice 12 §2, §5): the npm arguments, run in the plugin's real directory, the command +// overridable; a failure and an abort reject; git dependencies with a ref, a plugin inside a repository of several +// reached by path; plugins installed from git and npm (a package's chord.plugins manifest, the refusals, claims and +// cleanup). Git repositories and tarballs are local: no network. + +import { execFileSync } from "node:child_process"; +import { mkdir, readdir, readFile, realpath, symlink, utimes } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { + buildFacet, + discoverPlugins, + installDependencies, + installPlugin, +} from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { gitRepository, KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** A fake npm (a Node script) that records its arguments and directory, and exits with `code` after `delayMs`. */ +async function fakeNpm(dir: string, code: number, delayMs = 0): Promise { + const script = join(dir, "fake-npm.mjs"); + await writeFiles(dir, { + "fake-npm.mjs": [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(join(dir, "record.json"))}, JSON.stringify({ args: process.argv.slice(2), cwd: process.cwd() }));`, + `setTimeout(() => { if (${code} !== 0) console.error("npm said no"); process.exit(${code}); }, ${delayMs});`, + ].join("\n"), + }); + return [process.execPath, script]; +} + +describe("installDependencies", () => { + it("runs npm install without dev dependencies, peers, scripts, audit or funding, in the plugin's real directory", async () => { + const tools = await tempRoot(); + const elsewhere = await tempRoot(); + const root = await tempRoot(); + const dir = await writePlugin( + elsewhere, + "deps", + "@test/deps", + { "worker.ts": "" }, + { dependencies: { a: "1.0.0" } }, + ); + await symlink(dir, join(root, "deps")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context, { npmCommand: await fakeNpm(tools, 0) }); + expect(JSON.parse(await readFile(join(tools, "record.json"), "utf8"))).toEqual({ + args: ["install", "--omit=dev", "--legacy-peer-deps", "--ignore-scripts", "--no-audit", "--no-fund"], + cwd: await realpath(dir), + }); + }); + + it("rejects with the command's output when it fails, and when the context aborts", async () => { + const tools = await tempRoot(); + const root = await tempRoot(); + await writePlugin(root, "deps", "@test/deps", {}); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + expect( + await rejection(installDependencies(plugin, context, { npmCommand: await fakeNpm(tools, 1) })), + ).toMatchObject({ + message: expect.stringMatching(/^plugin "@test\/deps": .* install failed \(exit 1\): npm said no$/), + }); + const { context: aborted, cancel } = withCancel(context); + const running = installDependencies(plugin, aborted, { npmCommand: await fakeNpm(tools, 0, 5_000) }); + setTimeout(() => cancel(new Error("stop")), 50); + expect(await rejection(running)).toMatchObject({ name: "AbortError" }); + }); +}); + +/** The worker facet of plugin `name` under `root`, built for Node with Chord as the host package. */ +async function workerOf(root: string, name: string): Promise { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins.find((each) => each.name === name)!; + const options = { hostPackages: ["@earendil-works/chord/next"], platform: "node", plugins } as const; + const built = await buildFacet(plugin, "worker", options, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return built.source; +} + +/** A contracts package `@bob/notes` whose contract says `label`. */ +function notesPackage(label: string): Record { + return { + "package.json": JSON.stringify({ name: "@bob/notes", version: "1.0.0", exports: { ".": "./contract.ts" } }), + "contract.ts": `export const label = ${JSON.stringify(label)};`, + }; +} + +describe("installDependencies: git dependencies (slice 12 §2)", () => { + it("installs a git dependency pinned to a commit with npm; the build inlines that commit's file", async () => { + const first = await gitRepository("notes", notesPackage("notes at v1"), "v1"); + await gitRepository("notes", notesPackage("notes at v2"), "v2", first); + const root = await tempRoot(); + await writePlugin( + root, + "stats", + "@test/stats", + { "worker.ts": 'import { label } from "@bob/notes";\nexport default { setup: () => label };' }, + { dependencies: { "@bob/notes": `git+${first.url}#${first.commit}` } }, + ); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context); + const source = await workerOf(root, "@test/stats"); + expect(source).toContain("notes at v1"); + expect(source).not.toContain("notes at v2"); + }, 60_000); + + it("reaches a plugin inside a multi-plugin repository through a dependency on the whole repository, by path", async () => { + const repository = await gitRepository( + "plugins", + { + "package.json": JSON.stringify({ name: "@bob/plugins", version: "1.0.0", chord: { plugins: ["notes"] } }), + "notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "notes/contract.ts": 'export const label = "notes inside a repository of several";', + }, + "v1", + ); + const root = await tempRoot(); + await writePlugin( + root, + "stats", + "@test/stats", + { + "worker.ts": + 'import { label } from "@bob/plugins/notes/contract.ts";\nexport default { setup: () => label };', + }, + { dependencies: { "@bob/plugins": `git+${repository.url}#${repository.commit}` } }, + ); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context); + expect(await workerOf(root, "@test/stats")).toContain("notes inside a repository of several"); + }, 60_000); +}); + +describe("installPlugin (slice 12 §5)", () => { + it("clones a repository whose top is a plugin into /, at a tag, and installs its dependencies", async () => { + const repository = await gitRepository( + "hello", + { "package.json": JSON.stringify({ name: "@bob/hello" }), "worker.ts": "export default { setup: () => {} };" }, + "v1", + ); + const root = await tempRoot(); + const tools = await tempRoot(); + const npmCommand = await fakeNpm(tools, 0); + expect(await installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand })).toEqual([ + "@bob/hello", + ]); + expect(JSON.parse(await readFile(join(tools, "record.json"), "utf8")).cwd).toBe( + await realpath(join(root, "hello")), + ); + // No updating in place. + expect( + await rejection(installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand })), + ).toMatchObject({ + message: `installPlugin: ${join(root, "hello")} exists already`, + }); + }); + + it("clones a repository of several plugins (its subdirectories), pinned to a commit", async () => { + const first = await gitRepository( + "bob-plugins", + { + "notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "search/package.json": JSON.stringify({ name: "@bob/search" }), + }, + "v1", + ); + await gitRepository("bob-plugins", { "later/package.json": JSON.stringify({ name: "@bob/later" }) }, "v2", first); + const root = await tempRoot(); + const tools = await tempRoot(); + const names = await installPlugin({ git: first.url, ref: first.commit }, root, context, { + npmCommand: await fakeNpm(tools, 0), + }); + expect([...names].sort()).toEqual(["@bob/notes", "@bob/search"]); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@bob/notes", "@bob/search"]); + }); + + it("installs an npm package (here a local tarball) into /", async () => { + const source = await tempRoot(); + await writeFiles(source, { + "package.json": JSON.stringify({ name: "@bob/clock", version: "1.2.0" }), + "worker.ts": "export default { setup: () => {} };", + }); + const packed = execFileSync("npm", ["pack", "--silent", "--pack-destination", source], { + cwd: source, + encoding: "utf8", + }).trim(); + const root = await tempRoot(); + const names = await installPlugin({ npm: "@bob/clock", version: `file:${join(source, packed)}` }, root, context); + expect(names).toEqual(["@bob/clock"]); + expect(JSON.parse(await readFile(join(root, "@bob", "clock", "package.json"), "utf8"))).toMatchObject({ + version: "1.2.0", + }); + }, 60_000); + + it("installs exactly the folders a package's chord.plugins lists", async () => { + const repository = await gitRepository( + "bob-plugins", + { + "package.json": JSON.stringify({ + name: "@bob/plugins", + chord: { plugins: ["plugins/*", "!plugins/draft"] }, + }), + "plugins/notes/package.json": JSON.stringify({ name: "@bob/notes" }), + "plugins/draft/package.json": JSON.stringify({ name: "@bob/draft" }), + "tools/package.json": JSON.stringify({ name: "@bob/tools" }), + }, + "v1", + ); + const root = await tempRoot(); + const tools = await tempRoot(); + const names = await installPlugin({ git: repository.url, ref: "v1" }, root, context, { + npmCommand: await fakeNpm(tools, 0), + }); + expect(names).toEqual(["@bob/notes"]); + }); + + it("refuses a ref, a repository and a package name an option could hide in (Sol P1)", async () => { + const repository = await gitRepository("hello", { "package.json": JSON.stringify({ name: "@bob/hello" }) }, "v1"); + const root = await tempRoot(); + const marker = join(root, "..", "upload-pack-ran"); + for (const source of [ + { git: repository.url, ref: `--upload-pack=touch ${marker}` }, + { git: "--upload-pack=x", ref: "v1" }, + { npm: "--registry=http://evil", version: "1.0.0" }, + { npm: "x", version: "--foreground-scripts" }, + { npm: "../escaped", version: "1.0.0" }, + ]) { + expect(await rejection(installPlugin(source, root, context))).toMatchObject({ + message: expect.stringMatching(/^installPlugin: .* (is refused|is not an npm package name)$/), + }); + } + expect(await readdir(root)).toEqual([]); + expect(await readdir(join(root, "..")).then((names) => names.includes("upload-pack-ran"))).toBe(false); + }); + + it("refuses a plugin name another root has, and leaves nothing behind (Opus M4)", async () => { + const first = await tempRoot(); + const root = await tempRoot(); + await writePlugin(first, "zeta", "@test/same", {}); + const repository = await gitRepository("alpha", { "package.json": JSON.stringify({ name: "@test/same" }) }, "v1"); + const roots = [first, root]; + expect( + await rejection(installPlugin({ git: repository.url, ref: "v1" }, root, context, { roots })), + ).toMatchObject({ + message: `installPlugin: plugin "@test/same" is in ${join(await realpath(first), "zeta")} already`, + }); + expect(await readdir(root)).toEqual([]); + }); + + it("claims its directory first: of two concurrent installs one wins, the other removes nothing of it (Sol)", async () => { + const repository = await gitRepository("hello", { "package.json": JSON.stringify({ name: "@bob/hello" }) }, "v1"); + const root = await tempRoot(); + const tools = await tempRoot(); + const npmCommand = await fakeNpm(tools, 0); + const both = await Promise.allSettled([ + installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand }), + installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand }), + ]); + expect(both.map((each) => each.status).sort()).toEqual(["fulfilled", "rejected"]); + expect(both.find((each) => each.status === "rejected")).toMatchObject({ + reason: { message: `installPlugin: ${join(root, "hello")} exists already` }, + }); + expect(await readdir(join(root, "hello"))).toContain("package.json"); + }); + + it("removes the scope directory it made when a scoped npm install fails (Opus L2)", async () => { + const root = await tempRoot(); + const tools = await tempRoot(); + expect( + await rejection( + installPlugin({ npm: "@bob/notes", version: "1.0.0" }, root, context, { + npmCommand: await fakeNpm(tools, 1), + }), + ), + ).toMatchObject({ message: expect.stringContaining("pack @bob/notes@1.0.0 failed") }); + expect(await readdir(root)).toEqual([]); + }); + + it("removes staging directories an install crashed out of, once an hour old (Opus N6)", async () => { + const repository = await gitRepository("hello", { "package.json": JSON.stringify({ name: "@bob/hello" }) }, "v1"); + const root = await tempRoot(); + await mkdir(join(root, ".chord-install-old")); + await mkdir(join(root, ".chord-install-recent")); + const hourAgo = new Date(Date.now() - 2 * 60 * 60 * 1000); + await utimes(join(root, ".chord-install-old"), hourAgo, hourAgo); + const tools = await tempRoot(); + await installPlugin({ git: repository.url, ref: "v1" }, root, context, { npmCommand: await fakeNpm(tools, 0) }); + expect((await readdir(root)).sort()).toEqual([".chord-install-recent", "hello"]); + }); + + it("removes what arrived when it holds no plugin", async () => { + const repository = await gitRepository("empty", { "readme.md": "" }, "v1"); + const root = await tempRoot(); + expect(await rejection(installPlugin({ git: repository.url, ref: "v1" }, root, context))).toMatchObject({ + message: expect.stringMatching(/^installPlugin: .*empty: no package\.json$/), + }); + expect(await readdir(root)).toEqual([]); + }); +}); diff --git a/packages/chord/test/next-node/plugins/plugins.test.ts b/packages/chord/test/next-node/plugins/plugins.test.ts new file mode 100644 index 000000000..2aa3854de --- /dev/null +++ b/packages/chord/test/next-node/plugins/plugins.test.ts @@ -0,0 +1,816 @@ +// Tests of the plugins component (slice 7c §4, §5, §7): per plugin at startup and reload (a failing build, evaluation, +// setup; unchanged plugins untouched; added and removed plugins; the closure rule through builds), `rollout` (order, +// stopping at a rejecting step, marks), installs through the component, the app's API and required roots (slice 12), GC +// over 200 reloads, and end to end a server and a worker loading plugin directories, an edit rolled out, a broken edit +// kept old on both hosts. + +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Host, type ReloadReport } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + createPlugins, + type PluginError, + type Plugins, + type RolloutHost, + type RolloutStep, +} from "../../../src/next-node/plugins/index.ts"; +import { + changesOf, + delivered, + expectCollected, + rejection, + settledHeap, + transported, + valuesOf, +} from "../../next/support.ts"; +import { gitRepository, HOST_PACKAGES, kindsOf, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +/** The fixtures' contracts, as the `contracts` fixture declares them. */ +const Worker = hostKind("test.worker"); +const Notes = remote("test.notes", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { main: { state: { label: str(), count: int() }, methods: { add: method(undefined, int()) } } }, +}); +const Index = local("test.index", { version: 1, params: {}, members: { label: str() } }); +const Item = local("test.item", { version: 1, params: {}, members: { label: str() } }); + +/** A plugin `dir` named `@test/` serving `Item` labelled `label` from its worker facet. */ +async function itemPlugin(root: string, dir: string, label: string): Promise { + await writePlugin(root, dir, `@test/${dir}`, { + "worker.ts": [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + `const Item = local("test.${dir}", { version: 1, params: {}, members: { label: str() } });`, + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"), + }); +} + +/** The report of an in-process step's one host. */ +function reportOf(hosts: readonly RolloutHost[]): ReloadReport { + const [entry] = hosts; + if (entry === undefined || !("report" in entry)) throw new Error("no report"); + return entry.report; +} + +/** The component over `root`, its reports collected. */ +function component(root: string): { plugins: Plugins; reported: unknown[] } { + const reported: unknown[] = []; + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + onError: (e) => reported.push(e), + }, + context, + ); + return { plugins, reported }; +} + +/** The failures of a report as plugin names. */ +const failedNames = (report: { readonly failed: readonly { readonly plugin: string }[] }) => + report.failed.map(({ plugin }) => plugin); + +describe("per plugin at startup", () => { + it("leaves out a plugin whose build fails, whose evaluation throws or whose setup throws; loads the others and core", async () => { + const root = await tempRoot(); + await itemPlugin(root, "good", "good"); + await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); + await writePlugin(root, "throws", "@test/throws", { + "worker.ts": 'throw new Error("at import");\nexport default {};', + }); + await writePlugin(root, "setup", "@test/setup", { + "worker.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => { throw new Error("in setup"); });', + }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + const core = defineFacet((h) => h.serve(Item, () => ({ label: constant("core") }))); + const report = await plugins.load( + host, + { kind: "worker", core: [{ name: "@app/core", facet: core }], modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + [ + "@test/syntax", + expect.stringMatching(/^plugin "@test\/syntax": its worker facet failed to build:\nworker\.ts:1:/), + ], + ["@test/throws", "at import"], + ["@test/setup", 'plugin "@test/setup": in setup'], + ]); + const Good = local("test.good", { version: 1, params: {}, members: { label: str() } }); + expect((await host.get(Good, {}, context)).label.value).toBe("good"); + expect((await host.get(Item, {}, context)).label.value).toBe("core"); + await host.close(context); + plugins.close(); + }); +}); + +describe("per plugin at reload", () => { + /** A host loaded with plugins `a`, `b` and `c` (worker facets), and the component. */ + async function loaded() { + const root = await tempRoot(); + for (const dir of ["a", "b", "c"]) await itemPlugin(root, dir, `${dir}1`); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const label = async (dir: string) => + (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) + .label.value; + return { root, plugins, host, label }; + } + + it("rebuilds one plugin of three: only it changes; the others are passed as the same facets, untouched", async () => { + const { root, plugins, host, label } = await loaded(); + await itemPlugin(root, "b", "b2"); + const built = await plugins.build(["@test/b"], context); + expect(built.facets.map(({ plugin, facet }) => [plugin, facet])).toEqual([["@test/b", "worker"]]); + const report = await plugins.reload(host, built, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/b"], removed: [] }); + expect(report.objects).toEqual([{ object: "test.b", kind: "swapped" }]); + expect([await label("a"), await label("b"), await label("c")]).toEqual(["a1", "b2", "c1"]); + // Built again unchanged: the same digest, nothing reloaded. + const again = await plugins.reload(host, await plugins.build(["@test/b"], context), context); + expect(again.plugins).toEqual({ added: [], changed: [], removed: [] }); + await host.close(context); + plugins.close(); + }); + + it("keeps the loaded facet of a plugin whose build fails or whose evaluation throws; the others apply", async () => { + const { root, plugins, host, label } = await loaded(); + await writeFiles(join(root, "a"), { "worker.ts": "export default {" }); + await writeFiles(join(root, "b"), { "worker.ts": 'throw new Error("at import");\nexport default {};' }); + await itemPlugin(root, "c", "c2"); + const report = await plugins.reload( + host, + await plugins.build(["@test/a", "@test/b", "@test/c"], context), + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/c"], removed: [] }); + expect(failedNames(report)).toEqual(["@test/a", "@test/b"]); + expect([await label("a"), await label("b"), await label("c")]).toEqual(["a1", "b1", "c2"]); + await host.close(context); + plugins.close(); + }); + + it("adds a plugin discovered since, removes one no longer there; an added plugin that fails is left out", async () => { + const { root, plugins, host, label } = await loaded(); + await itemPlugin(root, "d", "d1"); + await writePlugin(root, "e", "@test/e", { "worker.ts": "export default {" }); + await rm(join(root, "c"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const built = await plugins.build([], context); + expect(built.removed).toEqual(["@test/c"]); + const report = await plugins.reload(host, built, context); + expect(report.plugins).toEqual({ added: ["@test/d"], changed: [], removed: ["@test/c"] }); + expect(failedNames(report)).toEqual(["@test/e"]); + expect(await label("d")).toBe("d1"); + await host.close(context); + plugins.close(); + }); + + it("keeps a consumer edited together with a provider whose new build fails (the closure rule through builds)", async () => { + const root = await tempRoot(); + const provider = (version: number) => + [ + 'import { constant, defineFacet, local, int } from "@earendil-works/chord/next";', + `export const P = local("test.provider", { version: ${version}, params: {}, members: { n: int() } });`, + `export default defineFacet((host) => host.serve(P, () => ({ n: constant(${version}) })));`, + ].join("\n"); + const consumer = (version: number) => + [ + 'import { defineFacet, local, int, str } from "@earendil-works/chord/next";', + `const P = local("test.provider", { version: ${version}, params: {}, members: { n: int() } });`, + 'const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } });', + `export default defineFacet((host) => host.serve(C, (_, self) => ({ label: self.derive([self.get(P, {}).n], ([n]) => "v${version} " + n) })));`, + ].join("\n"); + await writePlugin(root, "provider", "@test/provider", { "worker.ts": provider(1) }); + await writePlugin(root, "consumer", "@test/consumer", { "worker.ts": consumer(1) }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } }); + const shown = await host.get(C, {}, context); + await writeFiles(join(root, "provider"), { "worker.ts": `${provider(2)}\nexport const broken = ;` }); + await writeFiles(join(root, "consumer"), { "worker.ts": consumer(2) }); + const report = await plugins.reload( + host, + await plugins.build(["@test/provider", "@test/consumer"], context), + context, + ); + expect(failedNames(report)).toEqual(["@test/provider", "@test/consumer"]); + expect((report.failed[1]!.error as Error).message).toBe( + 'plugin "@test/consumer": needs test.provider v2, which plugin "@test/provider" serves on its old code', + ); + expect(shown.label.value).toBe("v1 1"); + await host.close(context); + plugins.close(); + }); +}); + +describe("rollout", () => { + /** A step recording its name into `order`, answering `report` for one host (or `hosts`), or rejecting with `error`. */ + const step = ( + name: string, + order: string[], + outcome: { report?: ReloadReport; hosts?: readonly RolloutHost[]; error?: Error }, + kinds: readonly string[] = ["worker"], + ): RolloutStep => ({ + name, + kinds, + reload: async () => { + order.push(name); + if (outcome.error !== undefined) throw outcome.error; + return { hosts: outcome.hosts ?? [{ host: name, report: outcome.report! }] }; + }, + }); + const report = (failed: readonly string[]): ReloadReport => ({ + plugins: { added: [], changed: [], removed: [] }, + failed: failed.map((plugin) => ({ plugin, error: new Error("x") })), + objects: [], + }); + + it("runs the steps in order, clears the marks of plugins applied on every step, keeps a plugin failed on any", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// a\n"); + await writeFile(join(root, "b", "worker.ts"), "// b\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a", "@test/b"] }), { + timeout: 5_000, + }); + const built = await plugins.build(["@test/a", "@test/b"], context); + const order: string[] = []; + const result = await plugins.rollout( + [step("server", order, { report: report([]) }), step("worker", order, { report: report(["@test/b"]) })], + built, + context, + ); + expect([order, result.stopped, result.steps.map(({ name }) => name)]).toEqual([ + ["server", "worker"], + undefined, + ["server", "worker"], + ]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/b"] }); + plugins.close(); + }); + + it("counts a host's error as the step's built plugins failing there; a host stopped cleanly dirties nothing (D12)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// a\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const stopped: RolloutHost[] = [{ host: "session|S1", stopped: true }]; + await plugins.rollout( + [step("workers", [], { hosts: stopped })], + await plugins.build(["@test/a"], context), + context, + ); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + await writeFile(join(root, "a", "worker.ts"), "// a again\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const crashed: RolloutHost[] = [ + { host: "session|S1", report: report([]) }, + { host: "session|S2", error: new Error("crashed") }, + ]; + const result = await plugins.rollout( + [step("workers", [], { hosts: crashed })], + await plugins.build(["@test/a"], context), + context, + ); + expect(result.steps).toEqual([{ name: "workers", hosts: crashed }]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + plugins.close(); + }); + + it("keeps dirty a plugin whose built kind no step shipped", async () => { + const root = await tempRoot(); + await writePlugin(root, "a", "@test/a", { "worker.ts": "export default { setup: () => {} };", "server.ts": "" }); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "export default { setup: () => {} }; // edited\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + await plugins.rollout([step("worker", [], { report: report([]) })], built, context); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + await plugins.rollout( + [step("worker", [], { report: report([]) }), step("server", [], { report: report([]) }, ["server"])], + built, + context, + ); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + plugins.close(); + }); + + it("stops at a step that rejects, clearing no mark; a change during the build keeps its mark", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// 1\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + const order: string[] = []; + const stopped = await plugins.rollout( + [step("server", order, { error: new Error("closed") }), step("worker", order, { report: report([]) })], + built, + context, + ); + expect([order, stopped.stopped?.step, (stopped.stopped?.error as Error).message]).toEqual([ + ["server"], + "server", + "closed", + ]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + // Saved again after the build: applied, the mark stays (the build did not include the new content). + const later = await plugins.build(["@test/a"], context); + await writeFile(join(root, "a", "worker.ts"), "// 2\n"); + await vi.waitFor(async () => { + await plugins.rollout([step("server", [], { report: report([]) })], later, context); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + }); + plugins.close(); + }); +}); + +describe("current sets (slice 10 D11)", () => { + it("scripts(kind): built on first use; a build changes no set; a rollout applies it right before a step of its kind", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await writePlugin(root, "b", "@test/b", { "server.ts": "export default { setup: () => {} };" }); + const { plugins } = component(root); + const digests = async () => + (await plugins.scripts("worker", context)).map(({ plugin, built }) => [ + plugin.name, + built.kind === "script" ? built.digest.slice(0, 8) : built.kind, + ]); + const first = await digests(); + expect(first.map(([name]) => name)).toEqual(["@test/a"]); + await itemPlugin(root, "a", "a2"); + const built = await plugins.build(["@test/a"], context); + expect(await digests()).toEqual(first); + const rejecting: RolloutStep = { + name: "server", + kinds: ["server"], + reload: async () => { + throw new Error("closed"); + }, + }; + const shipped: RolloutStep = { name: "workers", kinds: ["worker"], reload: async () => ({ hosts: [] }) }; + await plugins.rollout([rejecting, shipped], built, context); + expect(await digests()).toEqual(first); + await plugins.rollout([shipped], built, context); + expect(await digests()).not.toEqual(first); + plugins.close(); + }); +}); + +describe("installs through the component", () => { + it("clears the plugin's install mark and dirties it", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const reported: unknown[] = []; + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: (error) => reported.push(error), + }, + context, + ); + await plugins.build([], context); + await writeFile( + join(root, "a", "package.json"), + JSON.stringify({ name: "@test/a", type: "module", dependencies: {} }), + ); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ install: ["@test/a"] }), { timeout: 5_000 }); + await plugins.installDependencies("@test/a", context); + expect(plugins.state.value).toMatchObject({ install: [], dirty: ["@test/a"] }); + expect(reported).toEqual([]); + plugins.close(); + }); +}); + +describe("the app's API, required roots and installs (slice 12 §1, §5)", () => { + it("checks plugins' imports against the app's API, and its required root makes load reject", async () => { + const app = await tempRoot(); + const user = await tempRoot(); + const facet = (version: string) => + `import { defineFacet } from "@acme/app/${version}";\nexport default defineFacet(() => {});`; + await writePlugin(app, "core", "@test/core", { "worker.ts": facet("v3") }); + await writePlugin(app, "old", "@test/old", { "worker.ts": facet("v1") }); + await writePlugin(user, "mine", "@test/mine", { "worker.ts": facet("v1") }); + const modules = { "@acme/app/v3": { defineFacet } }; + const plugins = createPlugins( + { + roots: [{ path: app, required: true }, user], + api: { name: "@acme/app", versions: ["v3"] }, + kinds: kindsOf(modules), + onError: () => {}, + }, + context, + ); + const host = createHost({ onError: () => {} }); + const error = await rejection(plugins.load(host, { kind: "worker", core: [], modules }, context)); + expect(error).toMatchObject({ name: "PluginError" }); + expect((error as PluginError).required.map(({ plugin }) => plugin)).toEqual(["@test/old"]); + expect((error as PluginError).failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/mine", "@test/old"]); + expect(String((error as PluginError).required[0]!.error)).toContain( + '"@acme/app/v1": @acme/app/v1 is not available: this app provides v3', + ); + await host.close(context); + plugins.close(); + }); + + it("rejects load when a required root is missing, holds a broken package or loses a name to an earlier root (Opus M3)", async () => { + const facet = 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});'; + const broken = await tempRoot(); + await writePlugin(broken, "core", "@app/core", { "server.ts": facet }); + await writeFile(join(broken, "core", "package.json"), "{ not json"); + const user = await tempRoot(); + const app = await tempRoot(); + await writePlugin(user, "core", "@app/core", { "server.ts": facet }); + await writePlugin(app, "core", "@app/core", { "server.ts": facet }); + const cases = [ + { roots: [{ path: broken, required: true }], problem: /package\.json: / }, + { roots: [{ path: join(broken, "missing"), required: true }], problem: /the required root does not exist/ }, + { roots: [user, { path: app, required: true }], problem: /plugin "@app\/core" is in .* already/ }, + ]; + for (const { roots, problem } of cases) { + const plugins = createPlugins({ roots, kinds: kindsOf(HOST_PACKAGES), onError: () => {} }, context); + const host = createHost({ onError: () => {} }); + const error = await rejection( + plugins.load(host, { kind: "server", core: [], modules: HOST_PACKAGES }, context), + ); + expect(error).toMatchObject({ name: "PluginError" }); + expect((error as PluginError).problems.map(({ error: why }) => why)).toEqual([expect.stringMatching(problem)]); + await host.close(context); + plugins.close(); + } + }); + + it("installs a plugin into one of its roots and marks set; refuses a directory that is no root", async () => { + const root = await tempRoot(); + const repository = await gitRepository( + "hello", + { "package.json": JSON.stringify({ name: "@bob/hello" }), "worker.ts": "export default { setup: () => {} };" }, + "v1", + ); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: () => {}, + }, + context, + ); + await plugins.build([], context); + expect(await plugins.installPlugin({ git: repository.url, ref: "v1" }, root, context)).toEqual(["@bob/hello"]); + expect(plugins.state.value).toMatchObject({ set: true }); + const built = await plugins.build([], context); + expect(Object.keys(built.plugins)).toEqual(["@bob/hello"]); + expect(await rejection(plugins.installPlugin({ git: repository.url, ref: "v1" }, tools, context))).toMatchObject({ + message: `plugins.installPlugin: ${tools} is not one of the roots`, + }); + plugins.close(); + }); +}); + +describe("plugins whose files move, and removed plugins (slice 12 fix round 2, Sol)", () => { + it("rebuilds a single-file plugin that gains a same-named directory (the directory wins) without being named", async () => { + const root = await tempRoot(); + const facet = (label: string) => + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + 'const Item = local("test.hello", { version: 1, params: {}, members: { label: str() } });', + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"); + await writeFiles(root, { "hello.worker.ts": facet("from a file") }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const Hello = local("test.hello", { version: 1, params: {}, members: { label: str() } }); + expect((await host.get(Hello, {}, context)).label.value).toBe("from a file"); + await writePlugin(root, "hello", "hello", { "worker.ts": facet("from a directory") }); + const built = await plugins.build([], context); + expect(Object.keys(built.plugins)).toEqual(["hello"]); + await plugins.rollout([plugins.step("worker", host)], built, context); + expect((await host.get(Hello, {}, context)).label.value).toBe("from a directory"); + await host.close(context); + plugins.close(); + }); + + it("forgets what a removed plugin's builds read: a change there no longer dirties it", async () => { + const root = await tempRoot(); + await writePlugin(root, "contracts", "@test/contracts", { "contract.ts": "export const c = 1;" }); + await writePlugin(root, "user", "@test/user", { + "worker.ts": 'import { c } from "../contracts/contract.ts";\nexport default { setup: () => c };', + }); + const { plugins } = component(root); + await plugins.build(["@test/user"], context); + await rm(join(root, "user"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + // The removal rolled out (no host here): its mark clears. + await plugins.rollout([], await plugins.build([], context), context); + expect(plugins.state.value.dirty).toEqual([]); + await writeFile(join(root, "contracts", "contract.ts"), "export const c = 2;"); + await vi.waitFor(() => expect(plugins.state.value.dirty).toContain("@test/contracts"), { timeout: 5_000 }); + expect(plugins.state.value.dirty).not.toContain("@test/user"); + plugins.close(); + }); +}); + +describe("GC", () => { + it("collects old facets over 200 reloads; the heap stays flat after warm-up", async () => { + const root = await tempRoot(); + const made: WeakRef[] = []; + const probe = { made: (closure: object) => void made.push(new WeakRef(closure)) }; + const source = (label: string) => + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + 'import { made } from "@test/probe";', + 'const Item = local("test.gc", { version: 1, params: {}, members: { label: str() } });', + // A closure per evaluation, which only this evaluation's code holds. + `const state = { label: ${JSON.stringify(label)}, big: new Array(10_000).fill(${JSON.stringify(label)}) };`, + "made(state);", + "export default defineFacet((host) => host.serve(Item, () => ({ label: constant(state.label) })));", + ].join("\n"); + await writePlugin(root, "gc", "@test/gc", { "worker.ts": source("a") }); + const modules = { ...HOST_PACKAGES, "@test/probe": probe }; + const plugins = createPlugins({ roots: [root], kinds: kindsOf(modules), onError: () => {} }, context); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules }, context); + const first = await plugins.build(["@test/gc"], context); + await writeFiles(join(root, "gc"), { "worker.ts": source("b") }); + const second = await plugins.build(["@test/gc"], context); + const reload = async (count: number) => { + for (let i = 0; i < count; i++) await plugins.reload(host, i % 2 === 0 ? second : first, context); + }; + await reload(20); + const warm = await settledHeap(); + await reload(200); + const after = await settledHeap(); + expect(after - warm).toBeLessThan(2 * 1024 * 1024); + await expectCollected(...made.slice(0, made.length - 2)); + await host.close(context); + plugins.close(); + }); +}); + +describe("end to end: a server and a worker", () => { + it("loads plugin directories, rolls an edit out (server, then worker), and keeps a broken edit old on both", async () => { + type P = { readonly kind: "server" } | { readonly kind: "worker" }; + const root = await tempRoot(["contracts", "notes"]); + const { plugins } = component(root); + const [up, down] = memoryLinkPair(); + const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); + worker.connect(up, { principal: { kind: "server" } }, context); + const server = createHost

({ principal: { kind: "server" }, onError: () => {} }); + server.route( + Worker, + { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, + context, + ); + await plugins.load(worker, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await plugins.load(server, { kind: "server", core: [], modules: HOST_PACKAGES }, context); + const notes = await server.get(Notes, {}, context); + const index = await server.get(Index, {}, context); + expect([notes.label.value, index.label.value]).toEqual(["v1", "v1"]); + await notes.add(context); + const frames = changesOf(await notes.label.watch(context)); + const steps = [plugins.step("server", server as Host

), plugins.step("worker", worker as Host

)]; + + await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); + const rolled = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); + expect( + rolled.steps.map(({ name, hosts }) => [name, reportOf(hosts).plugins.changed, reportOf(hosts).objects]), + ).toEqual([ + ["server", ["@test/notes"], [{ object: "test.index", kind: "swapped" }]], + ["worker", ["@test/notes"], [{ object: "test.notes", kind: "swapped" }]], + ]); + await transported(); + await delivered(); + expect(valuesOf(frames)).toEqual(["v2"]); + expect([notes.label.value, notes.count.value, index.label.value]).toEqual(["v2", 1, "v2"]); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + + await writeFile(join(root, "notes", "label.ts"), "export const label = ;\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); + const broken = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); + expect(broken.steps.map(({ name, hosts }) => [name, failedNames(reportOf(hosts))])).toEqual([ + ["server", ["@test/notes"]], + ["worker", ["@test/notes"]], + ]); + await transported(); + expect([notes.label.value, index.label.value, notes.state.value]).toEqual(["v2", "v2", { kind: "live" }]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }); + await server.close(context); + await worker.close(context); + plugins.close(); + }); +}); + +describe("fix round 1", () => { + it("serializes reloads of one host: two started together keep both changes (Astra, Sol, Opus S1)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await itemPlugin(root, "a", "a2"); + const builtA = await plugins.build(["@test/a"], context); + await itemPlugin(root, "b", "b2"); + const builtB = await plugins.build(["@test/b"], context); + const reports = await Promise.all([plugins.reload(host, builtA, context), plugins.reload(host, builtB, context)]); + expect(reports.map(({ plugins: { changed } }) => changed)).toEqual([["@test/a"], ["@test/b"]]); + const label = async (dir: string) => + (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) + .label.value; + expect([await label("a"), await label("b")]).toEqual(["a2", "b2"]); + await host.close(context); + plugins.close(); + }); + + it("applies the removals of the build it was given, never a later discovery's (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + await itemPlugin(root, "a", "a2"); + const first = await plugins.build(["@test/a"], context); + await rm(join(root, "b"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const later = await plugins.build([], context); + expect([first.removed, later.removed]).toEqual([[], ["@test/b"]]); + const report = await plugins.reload(host, first, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/a"], removed: [] }); + expect((await plugins.reload(host, later, context)).plugins.removed).toEqual(["@test/b"]); + await host.close(context); + plugins.close(); + }); + + it("adopts what a reload committed when its wait aborts after the commit: a later reload keeps it (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const Item = local("test.a", { version: 1, params: {}, members: { label: str() } }); + const item = await host.get(Item, {}, context); + await itemPlugin(root, "a", "a2"); + const first = await plugins.build(["@test/a"], context); + // The abort lands right after the reload's commit (its publication batch), while it waits for its report. + const { batch } = host as ChordHost; + const commit = batch.commit.bind(batch); + const controller = new AbortController(); + const aborting = withAbortSignal(controller.signal, context); + const hook = vi.spyOn(batch, "commit").mockImplementation((adopt, ctx) => { + commit(adopt, ctx); + if (ctx === aborting) controller.abort(); + }); + expect(await rejection(plugins.reload(host, first, aborting))).toMatchObject({ + code: "aborted", + details: { committed: { added: [], changed: ["@test/a"], removed: [], failed: [] } }, + }); + hook.mockRestore(); + expect(item.label.value).toBe("a2"); + await itemPlugin(root, "b", "b2"); + await plugins.reload(host, await plugins.build(["@test/b"], context), context); + expect(item.label.value).toBe("a2"); + await host.close(context); + plugins.close(); + }); + + it("leaves a plugin out of a host whose kind its rebuilt facets no longer have (Opus recheck R1)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + // The plugin keeps a server facet only. + await rm(join(root, "a", "worker.ts")); + await writeFiles(join(root, "a"), { + "server.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});', + }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + expect(built.removed).toEqual([]); + expect((await plugins.reload(host, built, context)).plugins).toEqual({ + added: [], + changed: [], + removed: ["@test/a"], + }); + await host.close(context); + plugins.close(); + }); + + it("reloads a structured clone of a build as the build itself: it is plain data (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules: HOST_PACKAGES }, context); + const item = await host.get(local("test.a", { version: 1, params: {}, members: { label: str() } }), {}, context); + await itemPlugin(root, "a", "a2"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + expect((await plugins.reload(host, structuredClone(built), context)).plugins.changed).toEqual(["@test/a"]); + expect(item.label.value).toBe("a2"); + // The marks clear only through the build object itself (a copy carries no snapshot). + const result = await plugins.rollout([plugins.step("worker", host)], structuredClone(built), context); + expect(result.stopped).toBeUndefined(); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + await host.close(context); + plugins.close(); + }); + + it("reads a changed package.json again at the next build: a new dependency is accepted (Astra, Sol)", async () => { + const root = await tempRoot(); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const dir = await writePlugin(root, "a", "@test/a", { + "worker.ts": 'import { label } from "./helper.ts";\nexport default { setup: () => {}, label };', + "helper.ts": 'export const label = "old";', + }); + const plugins = createPlugins( + { + roots: [root], + kinds: kindsOf(HOST_PACKAGES), + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: () => {}, + }, + context, + ); + expect((await plugins.build([], context)).facets.map(({ kind }) => kind)).toEqual(["script"]); + await writeFiles(dir, { + "package.json": JSON.stringify({ name: "@test/a", type: "module", dependencies: { "new-dep": "1.0.0" } }), + }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ install: ["@test/a"] }), { timeout: 5_000 }); + await writePackage(dir, "new-dep", { "index.js": 'exports.label = "new";' }); + await writeFiles(dir, { "helper.ts": 'export { label } from "new-dep";' }); + await plugins.installDependencies("@test/a", context); + expect(plugins.state.value).toMatchObject({ set: false }); + const built = await plugins.build(["@test/a"], context); + expect(built.facets).toEqual([expect.objectContaining({ kind: "script", plugin: "@test/a" })]); + expect(plugins.discovery.plugins[0]!.dependencies).toEqual(["new-dep"]); + plugins.close(); + }); +}); + +describe("the reload queues and garbage collection", () => { + it("lets a rejected new facet go once its report is dropped, the kept old one live (Sol)", async () => { + const root = await tempRoot(); + const made: WeakRef[] = []; + const probe = { made: (value: object) => made.push(new WeakRef(value)) }; + const source = (bad: boolean) => + [ + 'import { constant, defineFacet, int, local } from "@earendil-works/chord/next";', + 'import { made } from "@test/probe";', + `const state = { bad: ${bad}, big: new Array(10000).fill("payload") };`, + "made(state);", + 'const Type = local("test.failed-gc", { version: 1, params: {}, members: { n: int() } });', + "export default defineFacet((h) => {", + '\tif (state.bad) throw new Error("bad setup");', + "\th.serve(Type, () => ({ n: constant(state.big.length) }));", + "});", + ].join("\n"); + const dir = await writePlugin(root, "failed", "@test/failed", { "worker.ts": source(false) }); + const modules = { ...HOST_PACKAGES, "@test/probe": probe }; + const plugins = createPlugins({ roots: [root], kinds: kindsOf(modules), onError: () => {} }, context); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [], modules }, context); + await writeFiles(dir, { "worker.ts": source(true) }); + const built = await plugins.build(["@test/failed"], context); + // The report (its failure's error holds the new facet's closure) is dropped here; the queues must not keep it. + expect((await plugins.reload(host, built, context)).failed.map(({ plugin }) => plugin)).toEqual(["@test/failed"]); + await expectCollected(made[1]!); + expect(made[0]!.deref()).toBeDefined(); + await host.close(context); + plugins.close(); + }); +}); diff --git a/packages/chord/test/next-node/plugins/runtime.test.ts b/packages/chord/test/next-node/plugins/runtime.test.ts new file mode 100644 index 000000000..d0115f276 --- /dev/null +++ b/packages/chord/test/next-node/plugins/runtime.test.ts @@ -0,0 +1,170 @@ +// Tests of the plugin runtime (slice 10 §3, D11): loading and reloading hosts with complete sets of shipped scripts +// (unchanged digests kept as the same facet, removed = loaded minus shipped, failed builds and evaluations kept old), +// a required root's failed plugin rejecting `load`, and that importing it loads no esbuild. + +import { execFileSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { + buildFacet, + createPluginRuntime, + discoverPlugins, + PluginError, + type PluginScript, +} from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { HOST_PACKAGES, tempRoot, writePlugin } from "./support.ts"; + +/** Writes plugin `@test/` serving `test.` labelled `label` from its worker facet (or `source` as it is). */ +async function plugin(root: string, dir: string, label: string, source?: string): Promise { + await writePlugin(root, dir, `@test/${dir}`, { + "worker.ts": + source ?? + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + `const Item = local("test.${dir}", { version: 1, params: {}, members: { label: str() } });`, + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"), + }); +} + +/** The worker scripts of every plugin under `root`, built now: what the server would ship. */ +async function shipped(root: string): Promise { + const { plugins } = await discoverPlugins([root], { kinds: ["worker"] }, context); + const options = { hostPackages: Object.keys(HOST_PACKAGES), platform: "node", plugins } as const; + return Promise.all( + plugins.map(async (each) => ({ plugin: each, built: await buildFacet(each, "worker", options, context) })), + ); +} + +/** The label `test.` shows on `host`. */ +async function label(host: ReturnType, dir: string): Promise { + const Item = local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }); + return (await host.get(Item, {}, context)).label.value; +} + +describe("loading and reloading with complete sets", () => { + it("loads a set and leaves out a plugin whose build or evaluation failed, reported with the host's report", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + await plugin(root, "broken", "", "export const = ;"); + await plugin(root, "throws", "", 'throw new Error("at import");'); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + const report = await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/broken", "@test/throws"]); + expect(await label(host, "a")).toBe("a1"); + await host.close(context); + }); + + it("reloads a complete set: same digests unchanged, a changed one applied, a loaded one not shipped removed", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + await plugin(root, "b", "b1"); + await plugin(root, "c", "c1"); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + await plugin(root, "b", "b2"); + const next = (await shipped(root)).filter(({ plugin }) => plugin.name !== "@test/c"); + const report = await runtime.reload(host, next, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/b"], removed: ["@test/c"] }); + expect([await label(host, "a"), await label(host, "b")]).toEqual(["a1", "b2"]); + await host.close(context); + }); + + it("keeps the loaded facet of a plugin whose new build or evaluation failed; an added failing one is left out", async () => { + const root = await tempRoot(); + await plugin(root, "a", "a1"); + const runtime = createPluginRuntime(); + const host = createHost({ onError: () => {} }); + await runtime.load( + host, + { kind: "worker", core: [], scripts: await shipped(root), modules: HOST_PACKAGES }, + context, + ); + await plugin(root, "a", "", "export const = ;"); + await plugin(root, "new", "", 'throw new Error("at import");'); + const report = await runtime.reload(host, await shipped(root), context); + expect(report.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/a", "@test/new"]); + expect(report.plugins).toEqual({ added: [], changed: [], removed: [] }); + expect(await label(host, "a")).toBe("a1"); + await host.close(context); + }); +}); + +describe("required roots at load (slice 12 §1)", () => { + it("rejects with a PluginError naming a required plugin that failed, after the host's load settled", async () => { + const app = await tempRoot(); + const user = await tempRoot(); + await plugin(app, "core", "core"); + await plugin(app, "broken", "", "export const = ;"); + await plugin(user, "flaky", "", 'throw new Error("at import");'); + const { plugins } = await discoverPlugins([{ path: app, required: true }, user], { kinds: ["worker"] }, context); + const options = { hostPackages: Object.keys(HOST_PACKAGES), platform: "node", plugins } as const; + const scripts = await Promise.all( + plugins.map(async (each) => ({ plugin: each, built: await buildFacet(each, "worker", options, context) })), + ); + const host = createHost({ onError: () => {} }); + const error = await rejection( + createPluginRuntime().load(host, { kind: "worker", core: [], scripts, modules: HOST_PACKAGES }, context), + ); + expect(error).toBeInstanceOf(PluginError); + const failure = error as PluginError; + expect(failure.message).toMatch( + /^required plugin failed at load: "@test\/broken": plugin "@test\/broken": its worker facet failed to build/, + ); + expect(failure.required.map(({ plugin }) => plugin)).toEqual(["@test/broken"]); + expect(failure.failed.map(({ plugin }) => plugin).sort()).toEqual(["@test/broken", "@test/flaky"]); + // The load settled: the required root's other plugin is loaded. + expect(await label(host, "core")).toBe("core"); + await host.close(context); + }); + + it("only reports a failed plugin of a root that is not required", async () => { + const user = await tempRoot(); + await plugin(user, "flaky", "", 'throw new Error("at import");'); + const host = createHost({ onError: () => {} }); + const report = await createPluginRuntime().load( + host, + { kind: "worker", core: [], scripts: await shipped(user), modules: HOST_PACKAGES }, + context, + ); + expect(report.failed.map(({ plugin }) => plugin)).toEqual(["@test/flaky"]); + await host.close(context); + }); +}); + +describe("the runtime's imports", () => { + it("loads no esbuild: a worker that only evaluates never imports it", () => { + const runtime = fileURLToPath(new URL("../../../src/next-node/plugins/runtime.ts", import.meta.url)); + // A hook refusing esbuild, then the runtime imported. + const script = [ + 'import { registerHooks } from "node:module";', + "registerHooks({ resolve(specifier, context, next) {", + ' if (specifier === "esbuild") throw new Error("esbuild imported");', + " return next(specifier, context);", + "} });", + `const { createPluginRuntime } = await import(${JSON.stringify(runtime)});`, + "createPluginRuntime();", + 'console.log("ok");', + ].join("\n"); + const output = execFileSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", "--input-type=module", "--eval", script], + { encoding: "utf8" }, + ); + expect(output.trim()).toBe("ok"); + }); +}); diff --git a/packages/chord/test/next-node/plugins/stack.test.ts b/packages/chord/test/next-node/plugins/stack.test.ts new file mode 100644 index 000000000..92ec48fc6 --- /dev/null +++ b/packages/chord/test/next-node/plugins/stack.test.ts @@ -0,0 +1,123 @@ +// Tests of stack traces (slice 7c §3, decision Q1 (b)): `mapStack` names plugin files and lines through each +// evaluated script's source map; nothing global; a script's map goes with its facet. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + type BuiltScript, + buildFacet, + discoverPlugins, + evaluateFacet, + mapStack, + type Plugin, +} from "../../../src/next-node/plugins/index.ts"; +import { expectCollected, settledHeap } from "../../next/support.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writePlugin } from "./support.ts"; + +/** A plugin `notes` whose worker facet's `fail` throws from `helper.ts` line 3. */ +async function failing(): Promise<{ script: BuiltScript; plugin: Plugin }> { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { + "worker.ts": 'import { helper } from "./helper.ts";\nexport default { setup: () => {}, fail: () => helper() };', + "helper.ts": + '// A helper.\nexport function helper(): never {\n\tthrow new Error("helper failed at " + Date.now());\n}', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet( + plugin, + "worker", + { hostPackages: Object.keys(HOST_PACKAGES), platform: "node" }, + context, + ); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return { script: built, plugin }; +} + +/** The error `facet`'s `fail` throws. */ +function thrownBy(facet: object): unknown { + try { + (facet as { readonly fail: () => void }).fail(); + } catch (error) { + return error; + } + throw new Error("fail did not throw"); +} + +describe("mapStack", () => { + it("maps a plugin script's positions to the plugin's files and lines; other lines stay; nothing global", async () => { + const { script, plugin } = await failing(); + const prepare = Error.prepareStackTrace; + const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + const error = thrownBy(facet); + const mapped = mapStack(error); + expect(mapped).toMatch(/at \S+ \(notes\/helper\.ts:3:8\)/); + expect(mapped).toMatch(/notes\/worker\.ts:2:\d+/); + expect(mapped).not.toContain("/chord-plugins/"); + // The raw stack is untouched: no `Error.prepareStackTrace` of the component's. + expect((error as Error).stack).toContain("/chord-plugins/@test/notes/worker.ts?"); + expect(Error.prepareStackTrace).toBe(prepare); + expect(mapStack("not an error")).toBe("not an error"); + }); + + it("lets a script's source map go with its facet", async () => { + const { script, plugin } = await failing(); + const error = await (async () => { + const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + return { error: thrownBy(facet), ref: new WeakRef(facet) }; + })(); + expect(mapStack(error.error)).toContain("notes/helper.ts:3:8"); + await expectCollected(error.ref); + // The finalization runs after the collection. + for (let round = 0; round < 10 && mapStack(error.error).includes("notes/helper.ts"); round++) await settledHeap(); + expect(mapStack(error.error)).toContain("/chord-plugins/@test/notes/worker.ts?"); + }); +}); + +describe("mapStack, fix round 1", () => { + it("maps an import-time throw (the map is registered before the script runs) (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "evaluation", "@test/evaluation", { + "worker.ts": + '// An import-time throw.\n\nthrow new Error("at import");\nexport default { setup: () => {} };\n', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet(plugin, "worker", { hostPackages: [], platform: "node" }, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + let failure: unknown; + try { + evaluateFacet(built, { hostPackages: {}, plugin }); + } catch (error) { + failure = error; + } + expect(mapStack(failure)).toContain("evaluation/worker.ts:3:"); + }); + + it("maps a throw while its default export is read (a getter) (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "getter", "@test/getter", { + "worker.ts": '// A getter failure.\nexport default {\n\tget setup() { throw new Error("getter boom"); } };\n', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet(plugin, "worker", { hostPackages: [], platform: "node" }, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + let failure: unknown; + try { + evaluateFacet(built, { hostPackages: {}, plugin }); + } catch (error) { + failure = error; + } + expect(mapStack(failure)).toContain("getter/worker.ts:3:"); + }); + + it("keeps a script's map while any evaluation of it lives: two hosts share it (Sol, Opus N3)", async () => { + const { script, plugin } = await failing(); + const first = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + const second = (() => new WeakRef(evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin })))(); + await expectCollected(second); + for (let round = 0; round < 5; round++) await settledHeap(); + expect(mapStack(thrownBy(first))).toContain("notes/helper.ts:3:8"); + }); +}); diff --git a/packages/chord/test/next-node/plugins/support.ts b/packages/chord/test/next-node/plugins/support.ts new file mode 100644 index 000000000..2d4b10d3d --- /dev/null +++ b/packages/chord/test/next-node/plugins/support.ts @@ -0,0 +1,104 @@ +// Test support for the plugins component (slice 7c, slice 12): fixture plugins copied into temporary roots, plugins, npm +// packages and local bare git repositories written there (no network), and the host packages the fixtures import. + +import { execFileSync } from "node:child_process"; +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach } from "vitest"; +import * as chordNext from "../../../src/next/index.ts"; +import type { HostPackages, KindDefinition } from "../../../src/next-node/plugins/index.ts"; + +/** The fixture plugins' directory. */ +export const FIXTURES = fileURLToPath(new URL("../fixtures/plugins/", import.meta.url)); + +/** The host packages the fixtures import: this process's Chord. */ +export const HOST_PACKAGES: HostPackages = { "@earendil-works/chord/next": chordNext }; + +/** The facet kinds of the fixtures' app. */ +export const KINDS = ["server", "worker", "web"] as const; + +/** The fixtures' app's kinds as the component takes them: Node servers and workers, a browser web, `modules`' names. */ +export function kindsOf(modules: HostPackages): Readonly> { + const hostPackages = Object.keys(modules); + return { + server: { platform: "node", hostPackages }, + worker: { platform: "node", hostPackages }, + web: { platform: "browser", hostPackages }, + }; +} + +const made: string[] = []; + +afterEach(async () => { + for (const dir of made.splice(0)) await rm(dir, { recursive: true, force: true }); +}); + +/** A temporary directory removed after the test; `fixtures` (names of fixture plugins) copied into it. */ +export async function tempRoot(fixtures: readonly string[] = []): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-plugins-")); + made.push(root); + for (const name of fixtures) await cp(join(FIXTURES, name), join(root, name), { recursive: true }); + return root; +} + +/** Writes `files` (relative path to text) into `dir`, creating directories. */ +export async function writeFiles(dir: string, files: Readonly>): Promise { + for (const [path, text] of Object.entries(files)) { + await mkdir(dirname(join(dir, path)), { recursive: true }); + await writeFile(join(dir, path), text); + } +} + +/** Writes a plugin named `name` into `root/dir`: its `package.json` (with `manifest`'s fields) and `files`. */ +export async function writePlugin( + root: string, + dir: string, + name: string, + files: Readonly>, + manifest: Readonly> = {}, +): Promise { + const path = join(root, dir); + await writeFiles(path, { "package.json": JSON.stringify({ name, type: "module", ...manifest }), ...files }); + return path; +} + +/** Writes an npm package `name` into `dir`'s `node_modules`: its `package.json` (CommonJS unless `type`) and files. */ +export async function writePackage( + dir: string, + name: string, + files: Readonly>, + manifest: Readonly> = {}, +): Promise { + const main = Object.keys(files)[0]!; + const json = JSON.stringify({ name, version: "1.0.0", main, ...manifest }); + await writeFiles(join(dir, "node_modules", name), { "package.json": json, ...files }); +} + +/** + * A local bare git repository `.git` in a temporary directory with `files` committed (one commit), tagged `tag`: + * its `file://` URL and the commit. A call with `previous` adds another commit on top of that repository. + */ +export async function gitRepository( + name: string, + files: Readonly>, + tag: string, + previous?: { readonly work: string; readonly bare: string }, +): Promise<{ readonly url: string; readonly commit: string; readonly work: string; readonly bare: string }> { + const git = (cwd: string, ...args: string[]) => + execFileSync("git", ["-c", "user.email=test@example.com", "-c", "user.name=test", ...args], { + cwd, + encoding: "utf8", + }).trim(); + const work = previous?.work ?? (await tempRoot()); + if (previous === undefined) git(work, "init", "--quiet"); + await writeFiles(work, files); + git(work, "add", "--all"); + git(work, "commit", "--quiet", "--message", tag); + git(work, "tag", tag); + const bare = previous?.bare ?? join(await tempRoot(), `${name}.git`); + if (previous === undefined) git(work, "clone", "--quiet", "--bare", work, bare); + else git(work, "push", "--quiet", "--tags", bare, "HEAD:refs/heads/master"); + return { url: `file://${bare}`, commit: git(work, "rev-parse", "HEAD"), work, bare }; +} diff --git a/packages/chord/test/next-node/plugins/testing.test.ts b/packages/chord/test/next-node/plugins/testing.test.ts new file mode 100644 index 000000000..60143d531 --- /dev/null +++ b/packages/chord/test/next-node/plugins/testing.test.ts @@ -0,0 +1,219 @@ +// Tests of testing a plugin alone (slice 12 §6): the plugin built as production builds it against the app's API +// modules, loaded on an in-memory host beside a stub serving what it uses; the registries it contributes to that nobody +// serves collected; the contracts it waits for unserved listed; a principal for the host and for a root scope. + +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { defineFacet, int, local, str } from "../../../src/next/index.ts"; +import { testPlugin } from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** The app's API v3 for the tests: Chord's surface re-exported, and a worker module with the app's `Panels`. */ +const Panels = chordNext.defineRegistry<{ readonly title: string }, unknown>("acme.panels", { version: 2 }); +const MODULES = { "@acme/app/v3": { ...chordNext }, "@acme/app/v3/worker": { Panels } }; +const API = { name: "@acme/app", versions: ["v3"] } as const; + +/** The notes contract the stats plugin uses, as the notes plugin declares it. */ +const Notes = local("acme.notes", { version: 1, params: { project: str() }, members: { words: int() } }); + +/** The stats plugin's contract, as a test of it declares it again. */ +const Stats = local("acme.stats", { version: 1, params: { project: str() }, members: { words: int() } }); + +/** A root with the stats plugin: serves `Stats` over `Notes`, contributes a panel, and a tag cloud over `Tags`. */ +async function statsPlugin(): Promise { + const root = await tempRoot(); + return writePlugin(root, "stats", "@acme/stats", { + "worker.ts": [ + 'import { constant, defineFacet, int, local, str } from "@acme/app/v3";', + 'import { Panels } from "@acme/app/v3/worker";', + 'const Notes = local("acme.notes", { version: 1, params: { project: str() }, members: { words: int() } });', + 'const Tags = local("acme.tags", { version: 1, params: {}, members: { list: str() } });', + 'const Stats = local("acme.stats", { version: 1, params: { project: str() }, members: { words: int() } });', + 'const Cloud = local("acme.cloud", { version: 1, params: {}, members: { list: str() } });', + "export default defineFacet((host, self) => {", + ' self.contribute(Panels, { title: "Word count" });', + " host.serve(Stats, ({ project }, self) => ({ words: self.get(Notes, { project }).words }));", + " host.serve(Cloud, (_params, self) => ({ list: self.get(Tags, {}).list }));", + "});", + ].join("\n"), + }); +} + +/** A stub of the notes plugin: every project has 42 words. */ +const notesStub = defineFacet((host) => host.serve(Notes, () => ({ words: chordNext.constant(42) }))); + +describe("testPlugin", () => { + it("loads the plugin built against the API beside a stub; collects its contributions and lists what it waits for", async () => { + const dir = await statsPlugin(); + const t = await testPlugin( + dir, + { + kind: "worker", + api: API, + modules: MODULES, + stubs: [{ name: "@acme/notes", facet: notesStub }], + principal: { kind: "user", id: "u1" }, + }, + context, + ); + const stats = await t.host.get(Stats, { project: "p1" }, context); + expect(stats.words.value).toBe(42); + expect(t.contributions(Panels).map(({ plugin, value }) => [plugin, value])).toEqual([ + ["@acme/stats", { title: "Word count" }], + ]); + expect(t.unserved()).toEqual(["acme.tags"]); + expect(await t.root(async (ctx) => ctx.principal, context)).toEqual({ kind: "user", id: "u1" }); + expect(await t.root(async (ctx) => ctx.principal, context, { principal: { kind: "user", id: "u2" } })).toEqual({ + kind: "user", + id: "u2", + }); + expect(() => t.contributions(chordNext.defineRegistry("acme.other", { version: 1 }))).toThrow( + "testPlugin: registry acme.other is not collected", + ); + await t.close(context); + }); + + it("rejects a build production would refuse, and a setup that throws", async () => { + const root = await tempRoot(); + const old = await writePlugin(root, "old", "@acme/old", { + "worker.ts": 'import { defineFacet } from "@acme/app/v1";\nexport default defineFacet(() => {});', + }); + expect(await rejection(testPlugin(old, { kind: "worker", api: API, modules: MODULES }, context))).toMatchObject({ + message: expect.stringContaining('"@acme/app/v1": @acme/app/v1 is not available: this app provides v3'), + }); + const throwing = await writePlugin(root, "throwing", "@acme/throwing", { + "worker.ts": + 'import { defineFacet } from "@acme/app/v3";\nexport default defineFacet(() => { throw new Error("no"); });', + }); + expect( + await rejection(testPlugin(throwing, { kind: "worker", api: API, modules: MODULES }, context)), + ).toMatchObject({ message: expect.stringMatching(/^testPlugin: setup failed: plugin "@acme\/throwing": no$/) }); + }); + + it("builds as production by default: the per-kind check active, api required with versioned modules (Opus M5)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "peek", "@acme/peek", { + "worker.ts": + 'import { defineFacet } from "@acme/app/v3";\nimport { panel } from "./contract.web.ts";\nexport default defineFacet(() => console.log(panel));', + "contract.web.ts": 'export const panel = "panel";', + }); + expect(await rejection(testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context))).toMatchObject({ + message: expect.stringContaining('"./contract.web.ts": a web file, imported into a worker facet'), + }); + expect(await rejection(testPlugin(dir, { kind: "worker", modules: MODULES }, context))).toMatchObject({ + message: 'testPlugin: modules hold the app\'s versioned API ("@acme/app/v3"): pass api, as production does', + }); + }); + + it("tests a single-file plugin and a plugin in a checkout of several (Opus L4)", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.worker.ts": [ + 'import { constant, defineFacet, local, str } from "@acme/app/v3";', + 'const Hello = local("acme.hello", { version: 1, params: {}, members: { text: str() } });', + 'export default defineFacet((host) => host.serve(Hello, () => ({ text: constant("hello") })));', + ].join("\n"), + }); + const single = await testPlugin( + join(root, "hello.worker.ts"), + { kind: "worker", api: API, modules: MODULES }, + context, + ); + const Hello = local("acme.hello", { version: 1, params: {}, members: { text: str() } }); + expect((await single.host.get(Hello, {}, context)).text.value).toBe("hello"); + await single.close(context); + // A checkout of several (its own package.json) beside another plugin of the root it copies from. + await writePlugin(root, "words", "@acme/words", { "contract.ts": 'export const WORDS = "words";' }); + await writeFiles(join(root, "repo"), { "package.json": JSON.stringify({ name: "@bob/repo" }) }); + const inner = await writePlugin(join(root, "repo"), "counter", "@bob/counter", { + "worker.ts": [ + 'import { constant, defineFacet, local, str } from "@acme/app/v3";', + 'import { WORDS } from "../../words/contract.ts";', + 'const Count = local("bob.count", { version: 1, params: {}, members: { text: str() } });', + "export default defineFacet((host) => host.serve(Count, () => ({ text: constant(WORDS) })));", + ].join("\n"), + }); + const checkout = await testPlugin(inner, { kind: "worker", api: API, modules: MODULES }, context); + const Count = local("bob.count", { version: 1, params: {}, members: { text: str() } }); + expect((await checkout.host.get(Count, {}, context)).text.value).toBe("words"); + await checkout.close(context); + }); + + it("collects until no new registry appears: an object live after the first collection contributes too (Sol)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "chain", "@acme/chain", { + "worker.ts": [ + 'import { constant, defineFacet, defineRegistry, local, str } from "@acme/app/v3";', + 'const A = defineRegistry("acme.a", { version: 1 });', + 'const B = defineRegistry("acme.b", { version: 1 });', + 'const Reader = local("acme.reader", { version: 1, params: {}, members: { s: str() } });', + "export default defineFacet((host, self) => {", + '\tself.contribute(A, "a");', + '\thost.serve(Reader, (_params, self) => { self.registry(A); self.contribute(B, "b"); return { s: constant("x") }; });', + "});", + ].join("\n"), + }); + const t = await testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context); + const B = chordNext.defineRegistry("acme.b", { version: 1 }); + expect(t.contributions(B).map(({ value }) => value)).toEqual(["b"]); + await t.close(context); + }); + + it("waits for slow activations a collection woke before it looks again (Opus R1)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "slow", "@acme/slow", { + "worker.ts": [ + 'import { constant, defineFacet, defineRegistry, local, str } from "@acme/app/v3";', + 'const A = defineRegistry("acme.a", { version: 1 });', + 'const B = defineRegistry("acme.b", { version: 1 });', + 'const Reader = local("acme.reader", { version: 1, params: {}, members: { s: str() } });', + "export default defineFacet((host, self) => {", + '\tself.contribute(A, "a");', + "\thost.serve(Reader, (_params, self) => {", + "\t\tself.registry(A);", + "\t\tself.open(() => new Promise((resolve) => setTimeout(resolve, 50)), () => {});", + '\t\tself.contribute(B, "b");', + '\t\treturn { s: constant("x") };', + "\t});", + "});", + ].join("\n"), + }); + const t = await testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context); + const B = chordNext.defineRegistry("acme.b", { version: 1 }); + expect(t.contributions(B).map(({ value }) => value)).toEqual(["b"]); + await t.close(context); + }); + + it("serves a registry contributed at a high version without an adapter per older version (Sol)", async () => { + const root = await tempRoot(); + const dir = await writePlugin(root, "high", "@acme/high", { + "worker.ts": + 'import { defineFacet, defineRegistry } from "@acme/app/v3";\nconst R = defineRegistry("acme.high", { version: 2000000 });\nexport default defineFacet((_host, self) => self.contribute(R, 1));', + }); + const started = performance.now(); + const t = await testPlugin(dir, { kind: "worker", api: API, modules: MODULES }, context); + expect(performance.now() - started).toBeLessThan(1_000); + const R = chordNext.defineRegistry("acme.high", { version: 1 }); + expect(t.contributions(R).map(({ value }) => value)).toEqual([1]); + await t.close(context); + }); + + it("takes no kind from test and declaration files (Opus N6)", async () => { + const root = await tempRoot(); + await writeFiles(root, { + "hello.worker.ts": 'import { defineFacet } from "@acme/app/v3";\nexport default defineFacet(() => {});', + "hello.test.ts": "", + "env.d.ts": "", + }); + const t = await testPlugin( + join(root, "hello.worker.ts"), + { kind: "worker", api: API, modules: MODULES }, + context, + ); + expect(Object.keys(t.plugin.facets)).toEqual(["worker"]); + await t.close(context); + }); +}); diff --git a/packages/chord/test/next-node/plugins/watcher.test.ts b/packages/chord/test/next-node/plugins/watcher.test.ts new file mode 100644 index 000000000..f54dcfaef --- /dev/null +++ b/packages/chord/test/next-node/plugins/watcher.test.ts @@ -0,0 +1,225 @@ +// Tests of the watcher (slice 7c §5, slice 12 §4): dirty marks from the files facets used and from plugin directories, +// `install`, `set`, `base`, a change during a build keeping its mark, `node_modules` and `.git` ignored, real file +// events, the marks as a JSON watchable, and single-file plugins' files. + +import { readdirSync, readFileSync } from "node:fs"; +import { realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { type BuiltFacet, discoverPlugins } from "../../../src/next-node/plugins/index.ts"; +import { PluginWatcher } from "../../../src/next-node/plugins/watcher.ts"; +import { changesOf, delivered } from "../../next/support.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** A root with plugins `a` (worker), `b` (server) and the contracts fixture, a base directory, and a watcher of both. */ +async function watched() { + const root = await realpath(await tempRoot(["contracts"])); + const base = await realpath(await tempRoot()); + await writePlugin(root, "a", "@test/a", { "worker.ts": "", "own.ts": "" }); + await writePlugin(root, "b", "@test/b", { "server.ts": "" }); + const failures: unknown[] = []; + const watcher = new PluginWatcher( + { roots: [root], base: [base], kinds: KINDS, failed: (error) => failures.push(error) }, + new Batch(() => {}, undefined), + ); + const discovery = await discoverPlugins([root], { kinds: KINDS }, context); + watcher.know(discovery, watcher.discoverySnapshot()); + const contract = join(root, "contracts", "contract.ts"); + const used = (plugin: string, facet: string, files: readonly string[]): BuiltFacet => ({ + kind: "script", + plugin, + facet, + source: "", + digest: "", + inputs: files.map((file) => ({ file, hash: "" })), + copied: [contract], + assets: [], + }); + watcher.used([ + used("@test/a", "worker", [join(root, "a", "worker.ts"), contract]), + used("@test/b", "server", [join(root, "b", "server.ts"), contract]), + ]); + return { root, base, watcher, failures }; +} + +describe("the watcher's marks", () => { + it("dirties every plugin whose facets used a changed file, and the plugin whose directory contains it", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "contracts", "contract.ts")); + expect(watcher.value).toEqual({ + dirty: ["@test/a", "@test/b", "@test/contracts"], + install: [], + set: false, + base: false, + }); + watcher.close(); + }); + + it("dirties the plugin of a file no facet read yet; asks for an install for a package.json", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "new.ts")); + watcher.mark(join(root, "b", "package.json")); + expect(watcher.value).toEqual({ dirty: ["@test/a"], install: ["@test/b"], set: false, base: false }); + // Installed: the mark clears, and the plugin is dirty (its packages changed). + watcher.installed("@test/b"); + expect(watcher.value).toEqual({ dirty: ["@test/a", "@test/b"], install: [], set: false, base: false }); + watcher.close(); + }); + + it("asks for discovery for a directory under a root and a facet file at a plugin's top; base for base code", async () => { + const { root, base, watcher } = await watched(); + watcher.mark(join(root, "c")); + expect(watcher.value).toMatchObject({ set: true, dirty: [] }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: false }); + watcher.mark(join(root, "b", "web.tsx")); + expect(watcher.value).toMatchObject({ set: true, dirty: ["@test/b"] }); + watcher.mark(join(base, "core.ts")); + expect(watcher.value).toMatchObject({ base: true }); + watcher.close(); + }); + + it("ignores node_modules and .git", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "node_modules", "x", "index.js")); + watcher.mark(join(root, "a", ".git", "HEAD")); + expect(watcher.value).toEqual({ dirty: [], install: [], set: false, base: false }); + watcher.close(); + }); + + it("keeps the mark of a change during a build: only an unchanged mark clears when applied", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "own.ts")); + const snapshot = watcher.snapshot(["@test/a"]); + // Saved again while the build runs. + watcher.mark(join(root, "a", "own.ts")); + watcher.applied(["@test/a"], snapshot); + expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }); + watcher.applied(["@test/a"], watcher.snapshot(["@test/a"])); + expect(watcher.value).toMatchObject({ dirty: [] }); + // A discovery that started before a change of the set keeps `set`. + const seen = watcher.discoverySnapshot(); + watcher.mark(join(root, "d")); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: true }); + watcher.close(); + }); + + it("marks from real file events, publishing the marks to a watch", async () => { + const { root, watcher } = await watched(); + const changes = changesOf(await watcher.watch(context)); + await writeFile(join(root, "a", "node_modules-free.ts"), "export {};"); + await writeFiles(join(root, "b", "node_modules", "pkg"), { "index.js": "" }); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + await delivered(); + expect(changes.at(-1)).toMatchObject({ kind: "value", value: { dirty: ["@test/a"] } }); + watcher.close(); + }); +}); + +describe("the watcher and single-file plugins (slice 12 §4)", () => { + it("dirties a single-file plugin for a change of its file; a file appearing or disappearing at a root's top sets set", async () => { + const { root, watcher } = await watched(); + await writeFiles(root, { "hello.server.ts": "" }); + watcher.mark(join(root, "hello.server.ts")); + expect(watcher.value).toMatchObject({ set: true, dirty: [] }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: false, dirty: [] }); + // Its own file: only it is dirty (not every plugin of the root its directory is). + watcher.mark(join(root, "hello.server.ts")); + expect(watcher.value).toMatchObject({ set: false, dirty: ["hello"] }); + await writeFiles(root, { "hello.web.tsx": "" }); + watcher.mark(join(root, "hello.web.tsx")); + expect(watcher.value).toMatchObject({ set: true }); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), watcher.discoverySnapshot()); + await rm(join(root, "hello.server.ts")); + watcher.mark(join(root, "hello.server.ts")); + expect(watcher.value).toMatchObject({ set: true, dirty: ["hello"] }); + watcher.close(); + }); +}); + +describe("the watcher, fix round 1", () => { + it("never lets an older build's applied snapshot clear a newer edit's mark (Astra)", async () => { + const { root, watcher } = await watched(); + const file = join(root, "a", "own.ts"); + watcher.mark(file); + const first = watcher.snapshot(["@test/a"]); + watcher.mark(file); + const second = watcher.snapshot(["@test/a"]); + watcher.applied(["@test/a"], second); + expect(watcher.value).toMatchObject({ dirty: [] }); + watcher.mark(file); + watcher.applied(["@test/a"], first); + expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }); + watcher.close(); + }); + + it("drops the install mark of a plugin the next discovery no longer has (example 38)", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "b", "package.json")); + expect(watcher.value).toMatchObject({ install: ["@test/b"] }); + await rm(join(root, "b"), { recursive: true }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ install: [] }); + watcher.close(); + }); + + it.skipIf(process.platform !== "linux")( + "puts no watch on node_modules or .git (Opus N2): 200 packages add no inotify watch", + async () => { + const root = await realpath(await tempRoot()); + const dir = await writePlugin(root, "a", "@test/a", { "worker.ts": "" }); + for (let i = 0; i < 200; i++) + await writeFiles(join(dir, "node_modules", `pkg${i}`, "lib"), { "index.js": "" }); + await writeFiles(join(dir, ".git", "objects"), { x: "" }); + const before = inotifyWatches(); + const watcher = new PluginWatcher( + { roots: [root], base: [], kinds: KINDS, failed: () => {} }, + new Batch(() => {}, undefined), + ); + // The root and the plugin's directory. + expect(inotifyWatches() - before).toBe(2); + watcher.close(); + }, + ); + + it("watches a directory that appears later, and marks the files in it", async () => { + const { root, watcher } = await watched(); + await writeFiles(join(root, "a", "lib"), { "first.ts": "" }); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.applied(["@test/a"], watcher.snapshot(["@test/a"])); + await writeFile(join(root, "a", "lib", "second.ts"), "export {};"); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.close(); + }); +}); + +describe("the watcher and links", () => { + it("survives a link loop created under a root (its stat never escapes the callback) and still marks it (Sol)", async () => { + const { root, watcher } = await watched(); + await symlink("loop", join(root, "a", "loop")); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.close(); + }); +}); + +/** The inotify watches this process holds (Linux): one line per watch in its inotify descriptors' fdinfo. */ +function inotifyWatches(): number { + let count = 0; + for (const fd of readdirSync("/proc/self/fdinfo")) { + try { + count += readFileSync(`/proc/self/fdinfo/${fd}`, "utf8") + .split("\n") + .filter((line) => line.startsWith("inotify wd:")).length; + } catch { + // Closed meanwhile. + } + } + return count; +} diff --git a/packages/chord/test/next-node/react/hooks.test.ts b/packages/chord/test/next-node/react/hooks.test.ts new file mode 100644 index 000000000..01f60197f --- /dev/null +++ b/packages/chord/test/next-node/react/hooks.test.ts @@ -0,0 +1,45 @@ +// Tests of the React bindings (slice 11 §3) in a real browser (Chromium; skipped without one), on the fixture's page: +// `useHold` holds while mounted and releases on unmount, `useWatch` re-renders on change, `useRefState` follows the +// reference's state, `useRegistry` the registry (the panels). + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { Browser, CHROMIUM } from "../web/browser.ts"; +import { freePort, panelPlugin, signIn, versionedEntry, webApp } from "../web/support.ts"; + +describe.skipIf(CHROMIUM === undefined)("React bindings", () => { + let browser: Browser; + + beforeAll(async () => { + browser = await Browser.start(); + }, 120_000); + + afterAll(async () => { + await browser?.close(); + }); + + it("hold while mounted, re-render on change, release on unmount", async () => { + const { app } = await webApp(await versionedEntry("1"), await freePort(), { + plugins: { notes: panelPlugin("Notes", "#123456") }, + }); + try { + await signIn(browser, app.ownerLink()); + // useRegistry: the plugin's panel. + await browser.wait('document.querySelector(".panel")?.textContent === "Notes"'); + // useHold + useRefState: the counter held and live. + await browser.wait('document.getElementById("count")?.dataset.state === "live"'); + expect(await browser.evaluate('document.getElementById("count").textContent')).toBe("0"); + // useWatch: a change re-renders. + await browser.evaluate('document.getElementById("count").click()'); + await browser.wait('document.getElementById("count").textContent === "1"'); + // Unmounted: the reference is released. + await browser.evaluate('document.getElementById("toggle").click()'); + await browser.wait('document.getElementById("count") === null && globalThis.counterState() === "ended"'); + // Mounted again: held again. + await browser.evaluate('document.getElementById("toggle").click()'); + await browser.wait('document.getElementById("count")?.dataset.state === "live"'); + } finally { + await app.close(context); + } + }, 120_000); +}); diff --git a/packages/chord/test/next-node/react/stores.test.ts b/packages/chord/test/next-node/react/stores.test.ts new file mode 100644 index 000000000..4e20490ed --- /dev/null +++ b/packages/chord/test/next-node/react/stores.test.ts @@ -0,0 +1,114 @@ +// Tests of the external stores the React bindings read (slice 11 §3), without React: a store's snapshot is the +// watchable's value (`undefined` when it has none, never a throw), and subscribing calls back on every change until +// unsubscribed. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + constant, + createHost, + defineFacet, + hostKind, + local, + memoryLinkPair, + method, + remote, + str, +} from "../../../src/next/index.ts"; +import { watchStore } from "../../../src/next-react/stores.ts"; + +const Notes = local("test.notes", { version: 1, params: {}, members: { text: str(), write: method(str()) } }); + +describe("a watch store", () => { + it("reads the value and calls back on changes until unsubscribed", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { + name: "@test/notes", + facet: defineFacet((h) => + h.serve(Notes, (_params, self) => { + const text = self.writable("a"); + return { + text: self.derive([text], ([value]) => value), + write: async (next) => { + text.set(next); + return null; + }, + }; + }), + ), + }, + ], + context, + ); + const notes = await host.get(Notes, {}, context); + const store = watchStore(notes.text); + expect(store.snapshot()).toBe("a"); + let calls = 0; + const unsubscribe = store.subscribe(() => calls++); + await new Promise((resolve) => setTimeout(resolve, 10)); + await notes.write("b", context); + for (let index = 0; index < 100 && store.snapshot() !== "b"; index++) await new Promise((r) => setTimeout(r, 5)); + expect(store.snapshot()).toBe("b"); + const seen = calls; + expect(seen).toBeGreaterThan(0); + unsubscribe(); + await notes.write("c", context); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(calls).toBe(seen); + notes.release(); + await host.close(context); + }); + + it("calls back once a held remote reference binds after the snapshot was read", async () => { + const Server = hostKind("server"); + const Greeting = remote("test.greeting", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { text: str() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [ + { + name: "@test/greeting", + facet: defineFacet((h) => + h.serve(Greeting, () => ({ variant: "main", members: { text: constant("hi") } })), + ), + }, + ], + context, + ); + const client = createHost({ onError: () => {} }); + await client.load([], context); + const held = client.hold(Greeting, {}, context); + const store = watchStore(held.text); + expect(store.snapshot()).toBeUndefined(); + let calls = 0; + const unsubscribe = store.subscribe(() => calls++); + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user" } }, context); + client.connect(mine, { principal: { kind: "server" } }, context); + for (let index = 0; index < 200 && calls === 0; index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(calls).toBeGreaterThan(0); + expect(store.snapshot()).toBe("hi"); + unsubscribe(); + held.release(); + await client.close(context); + await server.close(context); + }); + + it("gives undefined for no watchable and for one without a value", () => { + expect(watchStore(undefined).snapshot()).toBeUndefined(); + expect(watchStore(undefined).subscribe(() => {})).toBeTypeOf("function"); + const failing = { + get value(): string { + throw new Error("not ready"); + }, + watch: () => Promise.reject(new Error("x")), + }; + expect(watchStore(failing).snapshot()).toBeUndefined(); + }); +}); diff --git a/packages/chord/test/next-node/surfaces/tui.test.ts b/packages/chord/test/next-node/surfaces/tui.test.ts new file mode 100644 index 000000000..5ae2cfa45 --- /dev/null +++ b/packages/chord/test/next-node/surfaces/tui.test.ts @@ -0,0 +1,56 @@ +// Tests of the terminal surface's server side (slice 11 §4), through the app kit: the start a launcher downloads (signed +// in only), the base hash (public), the base bundle (an ES module answering builtins for bundled CommonJS) and the +// plugin scripts registering themselves. A terminal plugin has no assets: a Node build refuses them (Opus L5). + +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { TuiScripts } from "../../../src/next/index.ts"; +import type { TuiStart } from "../../../src/next-node/surfaces/index.ts"; +import { send } from "../http/support.ts"; +import { writePlugin } from "../plugins/support.ts"; +import { tuiApp } from "../tui/support.ts"; + +describe("the terminal surface", () => { + it("serves a signed-in launcher its start, the base bundle and the scripts; the base hash to anyone", async () => { + const { app, port, root } = await tuiApp(); + try { + const headers = { authorization: `Bearer ${(await readFile(app.ownerTokenFile, "utf8")).trim()}` }; + expect((await send(port, "GET", "/_chord/tui/start")).status).toBe(401); + const start = JSON.parse( + (await send(port, "GET", "/_chord/tui/start", { headers })).body.toString(), + ) as TuiStart; + expect(start).toEqual({ + base: app.tui!.base, + script: `/_chord/tui/base.${app.tui!.base}.js`, + combined: expect.stringMatching(/^\/_chord\/tui\/plugins\.[0-9a-f]{16}\.js$/), + }); + expect(JSON.parse((await send(port, "GET", "/_chord/tui/base")).body.toString())).toBe(app.tui!.base); + const base = (await send(port, "GET", start.script, { headers })).body.toString(); + // A CommonJS script (the sandbox gives it its own `require`), no ES module. + expect(base).toContain("require("); + expect(base).not.toMatch(/^\s*(?:import|export)\b/m); + const scripts = await app.host.get(TuiScripts, {}, context); + const [entry] = scripts.set.value.plugins; + expect(entry!.url).toBe(`/_chord/tui/plugin/@test~lines.${entry!.digest}.js`); + const script = (await send(port, "GET", entry!.url, { headers })).body.toString(); + expect(script.startsWith(`chordFacet("@test/lines","${entry!.digest}",`)).toBe(true); + expect((await send(port, "GET", start.combined, { headers })).body.toString()).toBe( + script.replace(/\n\/\/# sourceMappingURL=data:[^\n]*/, ""), + ); + scripts.release(); + // Terminal facets are Node builds: an asset import fails the build, so the surface serves none. + await writePlugin(root, "pictured", "@test/pictured", { + "tui.ts": 'import icon from "./icon.svg?url";\nexport default { setup: () => icon };', + "icon.svg": "", + }); + const built = await app.plugins.component.build(["@test/pictured"], context); + expect(built.facets.find(({ plugin }) => plugin === "@test/pictured")).toMatchObject({ + kind: "failed", + errors: [expect.stringContaining("assets by URL are for browser builds")], + }); + } finally { + await app.close(context); + } + }); +}); diff --git a/packages/chord/test/next-node/surfaces/web.test.ts b/packages/chord/test/next-node/surfaces/web.test.ts new file mode 100644 index 000000000..939cc731f --- /dev/null +++ b/packages/chord/test/next-node/surfaces/web.test.ts @@ -0,0 +1,218 @@ +// Tests of the web surface's server side (slice 11 §1), through the app kit: the base bundle (content-hashed, served +// forever, rebuilt at each start), plugin scripts as registered scripts and one combined file, `WebScripts` updated by +// a rollout, the page through the app's template with the baseline's CSP, signed in unless public. + +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { WebScripts } from "../../../src/next/index.ts"; +import { Sessions } from "../../../src/next-node/app/index.ts"; +import { pageScripts } from "../../../src/next-node/surfaces/index.ts"; +import { tempDir, testApp } from "../app/support.ts"; +import { send } from "../http/support.ts"; + +/** A web entry in a new directory: its path. */ +async function entry(text = "globalThis.started = true;\n"): Promise { + const dir = await tempDir(); + await writeFile(join(dir, "style.css"), ":root { --brand: red; }\n"); + await writeFile(join(dir, "main.ts"), `import "./style.css";\n${text}`); + return join(dir, "main.ts"); +} + +/** A web plugin's files: a facet with a stylesheet. */ +const WEB_PLUGIN = { + "web.ts": + 'import { defineFacet } from "@test/app/v1";\nimport "./notes.css";\nexport default defineFacet(() => {});', + "notes.css": ".notes { color: var(--accent); }", +}; + +/** The page template of these tests: the assets' tags in a minimal document. */ +const page = (request: { readonly nonce: string }, assets: Parameters[0]): string => + `t${pageScripts(assets, request.nonce)}
`; + +/** A kit app with the web surface and `plugins` (a web kind added). */ +async function webApp( + options: { + readonly plugins?: Record>; + readonly public?: boolean; + readonly main?: string; + } = {}, +) { + return testApp({ + kinds: { server: { platform: "node" }, worker: { platform: "node" }, web: { platform: "browser" } }, + plugins: options.plugins ?? {}, + web: { entry: options.main ?? (await entry()), page, ...(options.public === true ? { public: true } : {}) }, + }); +} + +/** A bearer header of a new owner session of `app`. */ +async function owner(app: Awaited>["app"]): Promise> { + const issued = await app.host.root( + async (ctx) => + (await ctx.get(Sessions, {})).issue({ principal: { kind: "owner" }, label: "t", kind: "device" }, ctx), + context, + ); + return { authorization: `Bearer ${issued.token}` }; +} + +/** The `src`s and `href`s of a page, in order. */ +function urlsOf(html: string): string[] { + return [...html.matchAll(/(?:src|href)="([^"]+)"/g)].map(([, url]) => url!); +} + +describe("the base bundle and the page", () => { + it("serves the page with the bootstrap, the scripts and the stylesheets; the files forever", async () => { + const { app, port } = await webApp({ plugins: { notes: WEB_PLUGIN } }); + const headers = await owner(app); + const answer = await send(port, "GET", "/", { headers }); + expect(answer.status).toBe(200); + const html = answer.body.toString(); + const nonce = /script-src [^;]*'nonce-([^']+)'/.exec(String(answer.headers["content-security-policy"]))![1]!; + expect(html).toContain(``, + }), + }, + // `:name` is one path segment, decoded. + "GET /notes/:name": { + public: true, + handler: async (request) => ({ kind: "json", value: { note: request.params.name } }), + }, + // A file of this plugin, saved under its name, with ranges. + "GET /notes/:name/download": { + public: true, + handler: async (request) => + fileResponse(pluginFile(`files/${request.params.name}.md`), { + download: `${request.params.name}.md`, + ranges: true, + }), + }, + }, + }), +); diff --git a/packages/chord/test/next/examples/41-workers/main.ts b/packages/chord/test/next/examples/41-workers/main.ts new file mode 100644 index 000000000..35460577b --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/main.ts @@ -0,0 +1,68 @@ +// Workers (slice 10): the server routes a session's address to its own worker process. The first reference to a +// session starts its worker; once nobody uses it for `idleMs`, the worker stops. The worker's code is the sessions +// plugin (`plugins/`), built by the server and shipped to the worker. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/41-workers/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Greeting, Session } from "./plugins/sessions/contract.ts"; + +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { worker: { platform: "node", hostPackages: ["@earendil-works/chord/next"] } }, + onError: console.error, + }, + context, +); + +// 1. The server, and its workers: one forked process per session, stopped after 300 ms without use. +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: console.error }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-41:${process.pid}:${address.key}`, + idleMs: () => 300, + plugins, + onError: console.error, + }, + context, +); +await server.load([], context); + +// 2. Every change of the workers' state, printed. +const watch = await workers.state.watch(context); +watch.start(async (call) => { + if (call.kind !== "value") return; + const entries = Object.entries(call.value).map(([key, state]) => `${key} ${state.status}`); + console.log(` workers: ${entries.join(", ") || "none"}`); +}); + +// 3. The first reference starts the worker; the greeting comes from that process. +console.log("1. getting S1's greeting"); +const greeting = await server.get(Greeting, { id: "S1" }, context); +console.log(`2. "${greeting.text.value}" (worker pid ${greeting.pid.value}, server pid ${process.pid})`); + +// 4. Releasing it leaves the worker idle: 300 ms later it stops. +greeting.release(); +console.log("3. released; waiting for the idle stop"); +while (Object.keys(workers.state.value).length > 0) { + await new Promise((resolve) => setTimeout(resolve, 50)); +} +console.log("4. stopped"); + +await watch.stop(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts b/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts new file mode 100644 index 000000000..0ef89bfff --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/contract.ts @@ -0,0 +1,13 @@ +// A session lives in its own worker process: `Session({ id })` is its address. + +import { hostKind, int, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** A session's greeting, served by the worker at the session's address. */ +export const Greeting = remote("example.greeting", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { text: str(), pid: int() } } }, +}); diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json b/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json new file mode 100644 index 000000000..3819a6f05 --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/sessions", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts b/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts new file mode 100644 index 000000000..79ee9efca --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/plugins/sessions/worker.ts @@ -0,0 +1,11 @@ +// The sessions plugin's worker facet: each worker greets with its process id. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Greeting } from "./contract.ts"; + +export default defineFacet((host) => + host.serve(Greeting, ({ id }) => ({ + variant: "main", + members: { text: constant(`hello from session ${id}`), pid: constant(process.pid) }, + })), +); diff --git a/packages/chord/test/next/examples/41-workers/worker.ts b/packages/chord/test/next/examples/41-workers/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/41-workers/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/42-worker-crash/main.ts b/packages/chord/test/next/examples/42-worker-crash/main.ts new file mode 100644 index 000000000..37297f924 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/main.ts @@ -0,0 +1,67 @@ +// A worker crash (slice 10): workers never restart by themselves. A crashed worker's address stays failed, its +// holders see `failed` with the error, and nothing forks until the app retries (a Retry button) or a rollout ships new +// code. Here: a session crashes, stays failed for a while, then Retry brings it back. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/42-worker-crash/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Process, Session } from "./plugins/sessions/contract.ts"; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { worker: { platform: "node", hostPackages: ["@earendil-works/chord/next"] } }, + onError: console.error, + }, + context, +); +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: () => {} }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-42:${process.pid}:${address.key}`, + plugins, + onError: (error) => console.log(` reported: ${error.kind}`), + }, + context, +); +await server.load([], context); + +// 1. A held session (a tab showing it): its worker runs. +const session = server.hold(Process, { id: "S1" }, context); +const state = () => session.state.value.kind; +while (state() !== "live") await sleep(10); +console.log(`1. S1 runs in process ${session.pid.value}`); + +// 2. It crashes. The address is failed, and the holder sees why; it stays on the address's path. +await session.crash(context); +while (state() !== "failed") await sleep(10); +const failed = workers.state.value["session|S1"]; +console.log(`2. crashed: ${failed?.status === "failed" ? failed.error.message : failed?.status}`); + +// 3. No restart loop: a second later it is still failed, and no other worker was started. +await sleep(1_000); +console.log(`3. a second later: ${workers.state.value["session|S1"]?.status}, the holder: ${state()}`); + +// 4. Retry (the app's button): the worker starts again and the same holder binds to it. +await workers.retry(Session({ id: "S1" }), context); +while (state() !== "live") await sleep(10); +console.log(`4. after Retry: S1 runs in process ${session.pid.value}`); + +session.release(); +await workers.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts new file mode 100644 index 000000000..0a995c248 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/contract.ts @@ -0,0 +1,13 @@ +// A session lives in its own worker process: `Session({ id })` is its address. + +import { hostKind, int, method, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** A session's process: its pid, and a way to make it crash. */ +export const Process = remote("example.process", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { pid: int() }, methods: { crash: method() } } }, +}); diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json new file mode 100644 index 000000000..3819a6f05 --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/sessions", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts new file mode 100644 index 000000000..cb8cae63c --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/plugins/sessions/worker.ts @@ -0,0 +1,17 @@ +// The sessions plugin's worker facet: its process id, and `crash`, which ends the process (a bug, an out-of-memory). + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Process } from "./contract.ts"; + +export default defineFacet((host) => + host.serve(Process, () => ({ + variant: "main", + members: { + pid: constant(process.pid), + crash: async () => { + setTimeout(() => process.exit(13), 10); + return null; + }, + }, + })), +); diff --git a/packages/chord/test/next/examples/42-worker-crash/worker.ts b/packages/chord/test/next/examples/42-worker-crash/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/42-worker-crash/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/43-rollout/main.ts b/packages/chord/test/next/examples/43-rollout/main.ts new file mode 100644 index 000000000..5fd443756 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/main.ts @@ -0,0 +1,85 @@ +// A rollout across the server and its workers (slice 10): an edit of a plugin is built once by the server and rolled +// out step by step, the server first, then every running worker; each host reports what changed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/43-rollout/main.ts +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { hostKind } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, type RolloutHost } from "../../../../src/next-node/plugins/index.ts"; +import { createWorkers } from "../../../../src/next-node/workers/index.ts"; +import { Banner, Label, Session } from "./plugins/notes/contract.ts"; + +/** Waits until `done()` holds (a watcher's mark, a frame from a worker), checking every 10 ms. */ +async function waitUntil(done: () => boolean): Promise { + while (!done()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** One line for what a rollout step did on one of its hosts. */ +function describe(host: RolloutHost): string { + if ("report" in host) return `changed ${host.report.plugins.changed.join(", ")}`; + if ("stopped" in host) return "stopped"; + return "failed"; +} + +// The example edits a plugin: it works on a copy of `plugins/`. +const root = await mkdtemp(join(tmpdir(), "chord-example-43-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +const node = { platform: "node", hostPackages: ["@earendil-works/chord/next"] } as const; +const plugins = createPlugins( + { roots: [root], kinds: { server: node, worker: node }, onError: console.error }, + context, +); +const server = createHost({ address: hostKind("server")(), principal: { kind: "server" }, onError: console.error }); +const workers = createWorkers( + server, + Session, + { + kind: "worker", + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + authorize: () => true, + principal: (address) => ({ kind: "worker", session: address.params.id }), + start: () => ({}), + lock: (address) => `example-43:${process.pid}:${address.key}`, + plugins, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [], modules: { "@earendil-works/chord/next": chordNext } }, context); + +// 1. The server's banner and two running sessions. +const banner = await server.get(Banner, {}, context); +const s1 = await server.get(Label, { id: "S1" }, context); +const s2 = await server.get(Label, { id: "S2" }, context); +console.log(`1. ${banner.text.value} | ${s1.text.value} | ${s2.text.value}`); + +// 2. An edit: the watcher marks the plugin; one build serves both kinds. +await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); +await waitUntil(() => plugins.state.value.dirty.includes("@example/notes")); +const built = await plugins.build(plugins.state.value.dirty, context); + +// 3. The rollout: the server, then the workers (each running one reloads in parallel). +const result = await plugins.rollout([plugins.step("server", server), workers.step("workers")], built, context); +for (const step of result.steps) { + for (const host of step.hosts) console.log(`2. ${step.name}: ${host.host} ${describe(host)}`); +} + +// 4. Every host shows the new label (the workers' frames arrive over their links). +await waitUntil(() => s2.text.value === "S2: v2"); +console.log(`3. ${banner.text.value} | ${s1.text.value} | ${s2.text.value}`); +console.log(`4. still dirty: ${JSON.stringify(plugins.state.value.dirty)}`); + +banner.release(); +s1.release(); +s2.release(); +await workers.close(context); +await server.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts new file mode 100644 index 000000000..86d192d7e --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/contract.ts @@ -0,0 +1,16 @@ +// The notes' contracts: a label on the server, and one in each session's worker. + +import { hostKind, local, remote, str } from "@earendil-works/chord/next"; + +export const Session = hostKind("session", { id: str() }); + +/** The label as the server shows it. */ +export const Banner = local("example.banner", { version: 1, params: {}, members: { text: str() } }); + +/** The label as a session's worker shows it. */ +export const Label = remote("example.label", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Session({ id }), + variants: { main: { state: { text: str() } } }, +}); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts new file mode 100644 index 000000000..c8c560a6e --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The label every host shows; the example edits it. + +export const label = "v1"; diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json b/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts new file mode 100644 index 000000000..ca8c22e00 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/server.ts @@ -0,0 +1,7 @@ +// The notes plugin's server facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Banner } from "./contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => host.serve(Banner, () => ({ text: constant(`server: ${label}`) }))); diff --git a/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts b/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts new file mode 100644 index 000000000..972e91925 --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/plugins/notes/worker.ts @@ -0,0 +1,9 @@ +// The notes plugin's worker facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Label } from "./contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(Label, ({ id }) => ({ variant: "main", members: { text: constant(`${id}: ${label}`) } })), +); diff --git a/packages/chord/test/next/examples/43-rollout/worker.ts b/packages/chord/test/next/examples/43-rollout/worker.ts new file mode 100644 index 000000000..2fb4fccaa --- /dev/null +++ b/packages/chord/test/next/examples/43-rollout/worker.ts @@ -0,0 +1,22 @@ +// The worker process: a host at the session's address, connected to the server, loading the plugins it was shipped. + +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { workerMain } from "../../../../src/next-node/workers/index.ts"; + +workerMain( + async ({ address, parent, plugins, onClose }, context) => { + const host = createHost({ address, principal: { kind: "worker" }, onError: console.error }); + onClose(() => host.close(BACKGROUND_CONTEXT)); + // Connect first: the server sees the worker started; then load. + host.connect(parent, { principal: { kind: "server" }, trusted: true }, context); + await plugins.load(host, { kind: "worker", core: [] }, context); + return {}; + }, + { + hostPackages: { "@earendil-works/chord/next": chordNext }, + redact: (error) => ({ message: String(error) }), + onError: console.error, + }, +); diff --git a/packages/chord/test/next/examples/44-http-wraps/main.ts b/packages/chord/test/next/examples/44-http-wraps/main.ts new file mode 100644 index 000000000..af970636c --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/main.ts @@ -0,0 +1,37 @@ +// Wraps (slice 10): plugins wrap routes, their own or everyone's, in phases (`observe`, `headers`, `transform`, +// outermost first). Here the notes plugin wraps its own route to add a header, and an audit plugin wraps every route +// to log it; the audit plugin needs no route of its own. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/44-http-wraps/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +for (const path of ["/notes", "/missing"]) { + console.log(`GET ${path}`); + const response = await fetch(`http://127.0.0.1:${port}${path}`); + console.log(` -> ${response.status}, Cache-Control: ${response.headers.get("cache-control")}`); +} +// The 404 has no route, so no wrap ran for it: the audit log shows only /notes, and it keeps the default +// `Cache-Control: no-store` that the notes wrap replaced on its own route. + +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json new file mode 100644 index 000000000..10a68118f --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/audit", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts new file mode 100644 index 000000000..e4259d470 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/audit/server.ts @@ -0,0 +1,24 @@ +// The audit plugin: no routes of its own, one wrap around every route (`scope: "all"`), outermost (`observe`). + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + wraps: [ + { + name: "log", + phase: "observe", + scope: "all", + wrap: async (request, next, ctx) => { + const response = await next(request); + const status = "status" in response ? response.status : 200; + console.log( + ` audit: ${ctx.principal === undefined ? "anonymous" : "someone"} ${request.method} ${request.path} ${status}`, + ); + return response; + }, + }, + ], + }), +); diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts new file mode 100644 index 000000000..db5441651 --- /dev/null +++ b/packages/chord/test/next/examples/44-http-wraps/plugins/notes/server.ts @@ -0,0 +1,22 @@ +// The notes plugin: a route, and its own wrap (around its own routes only) adding a header. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /notes": { public: true, handler: async () => ({ kind: "json", value: ["buy milk", "call mom"] }) }, + }, + wraps: [ + { + name: "cache", + phase: "headers", + wrap: async (request, next) => { + const response = await next(request); + return { ...response, headers: { ...response.headers, "Cache-Control": "max-age=60" } }; + }, + }, + ], + }), +); diff --git a/packages/chord/test/next/examples/45-plugin-socket/main.ts b/packages/chord/test/next/examples/45-plugin-socket/main.ts new file mode 100644 index 000000000..9f10cd8b8 --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/main.ts @@ -0,0 +1,64 @@ +// A plugin socket (slice 10): a plugin contributes a WebSocket endpoint for its own protocol (a terminal, voice). The +// server checks the origin and authenticates first: without a principal the upgrade is refused before plugin code. +// Here the app's authenticator reads a principal from a header (a real app: a signed cookie, `readCookie`). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/45-plugin-socket/main.ts +import { once } from "node:events"; +import { fileURLToPath } from "node:url"; +import { WebSocket } from "ws"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +type Principal = { readonly kind: "server" } | { readonly kind: "user"; readonly name: string }; +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +// The app's core facet: who someone is. +const app = defineFacet((_host, self) => { + self.contribute(chordHttp.Authenticators, (request) => { + const name = request.headers.get("x-user"); + return name === null ? undefined : { kind: "user", name }; + }); +}); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +const core = [ + { name: "chord.http", facet: http.facet }, + { name: "@example/app", facet: app }, +]; +await plugins.load(server, { kind: "server", core, modules }, context); +const { port } = await http.listening(context); + +// 1. Without a principal: refused (403), the plugin never sees it. +const refused = new WebSocket(`ws://127.0.0.1:${port}/echo/lobby`); +const [, response] = await once(refused, "unexpected-response"); +console.log(`1. anonymous: refused ${response.statusCode}`); + +// 2. As ada: the plugin's handler greets, then echoes. +const socket = new WebSocket(`ws://127.0.0.1:${port}/echo/lobby`, { headers: { "x-user": "ada" } }); +const messages: string[] = []; +socket.on("message", (data) => messages.push(String(data))); +await once(socket, "open"); +socket.send("hello"); +// The greeting and the echo. +while (messages.length < 2) { + await new Promise((resolve) => setTimeout(resolve, 10)); +} +for (const message of messages) console.log(`2. ${message}`); + +// 3. Closing the component closes its sockets (1001, going away); then the host. +const closed = once(socket, "close"); +await http.close(context); +await server.close(context); +const [code] = await closed; +console.log(`3. closed with ${code}`); +plugins.close(); diff --git a/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json new file mode 100644 index 000000000..ed606991b --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/echo", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts new file mode 100644 index 000000000..abf014eb3 --- /dev/null +++ b/packages/chord/test/next/examples/45-plugin-socket/plugins/echo/server.ts @@ -0,0 +1,16 @@ +// The echo plugin: a WebSocket of its own protocol at `/echo/:room`, after the server checked who is connecting. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + sockets: { + "/echo/:room": (socket, request, ctx) => { + const who = JSON.stringify(ctx.principal); + socket.send(`welcome to ${request.params.room}, ${who}`); + socket.on("message", (data) => socket.send(`echo: ${String(data)}`)); + }, + }, + }), +); diff --git a/packages/chord/test/next/examples/46-allowed-hosts/main.ts b/packages/chord/test/next/examples/46-allowed-hosts/main.ts new file mode 100644 index 000000000..67c346516 --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/main.ts @@ -0,0 +1,61 @@ +// Allowed hosts (slice 10): the server answers only requests on loopback or through a host a plugin contributes. A +// plugin can allow a host at runtime, a tunnel naming its public host, and take it back; links and sockets through a +// host no longer allowed are terminated at once. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/46-allowed-hosts/main.ts +import { request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { local, method, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +/** The tunnel plugin's object, as the app's code declares it (the same contract). */ +const Tunnel = local("example.tunnel", { version: 1, params: {}, members: { open: method(str()), close: method() } }); + +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); +const { port } = await http.listening(context); + +/** GET /hello through `host` (as a tunnel would forward it), printed. */ +async function hello(host: string): Promise { + const answer = await new Promise((resolve) => { + const req = request({ host: "127.0.0.1", port, path: "/hello", headers: { host } }, (res) => { + let body = ""; + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => resolve(`${res.statusCode} ${body}`)); + }); + req.end(); + }); + console.log(` GET /hello via ${host} -> ${answer}`); +} + +const tunnel = await server.get(Tunnel, {}, context); +console.log("1. no tunnel yet"); +await hello(`localhost:${port}`); +await hello("notes.example.net"); +console.log("2. the tunnel opens"); +await tunnel.open("notes.example.net", context); +await hello("notes.example.net"); +console.log("3. the tunnel closes"); +await tunnel.close(context); +await hello("notes.example.net"); + +tunnel.release(); +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json new file mode 100644 index 000000000..7681eec3f --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/tunnel", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts new file mode 100644 index 000000000..da5783e73 --- /dev/null +++ b/packages/chord/test/next/examples/46-allowed-hosts/plugins/tunnel/server.ts @@ -0,0 +1,36 @@ +// The tunnel plugin: when its tunnel opens, it allows the tunnel's public host (requests through it are https); when +// it closes, it takes the host back. Its contribution is a writable, so the router follows each change. + +import { defineFacet, local, method, str } from "@earendil-works/chord/next"; +import { Http, type HttpContribution } from "@earendil-works/chord/next/http"; + +export const Tunnel = local("example.tunnel", { + version: 1, + params: {}, + members: { open: method(str()), close: method() }, +}); + +export default defineFacet((host, self) => { + self.contribute(Http, { + routes: { + "GET /hello": { + public: true, + handler: async (request) => ({ kind: "json", value: { origin: request.origin } }), + }, + }, + }); + host.serve(Tunnel, (_params, tunnel) => { + const allowed = tunnel.writable({ hosts: [] }); + tunnel.contribute(Http, allowed); + return { + open: async (name) => { + allowed.set({ hosts: [name] }); + return null; + }, + close: async () => { + allowed.set({ hosts: [] }); + return null; + }, + }; + }); +}); diff --git a/packages/chord/test/next/examples/47-http-own-server/main.ts b/packages/chord/test/next/examples/47-http-own-server/main.ts new file mode 100644 index 000000000..6595b2517 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/main.ts @@ -0,0 +1,70 @@ +// Your own server (slice 10): the HTTP component is a Web-standard `fetch` handler, so an app with a server of its own +// mounts it instead of Chord's listener. Here a plain Node server converts every request with `nodeRequest` (the +// listener's checks: a malformed target refused, a second `Host` kept for admission to refuse; and the connection: +// loopback requests count as local only with it), passes it to `http.fetch`, and every upgrade to `http.upgrade`. +// With Hono the same is one line: `app.all("/*", (c) => http.fetch(c.req.raw, connection))`. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/47-http-own-server/main.ts +import { createServer, request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +// 1. The component, without a listener: no `listen` setting. +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { onError: console.error }, context); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +await plugins.load(server, { kind: "server", core: [{ name: "chord.http", facet: http.facet }], modules }, context); + +// 2. The app's own server: a Web `Request` in, a Web `Response` out (each header line kept: several `Set-Cookie`). A +// real app adds its own routes around it. +const own = createServer(async (message, res) => { + const converted = chordHttp.nodeRequest(message, res); + if (converted.kind === "refused") return void res.writeHead(converted.status).end(); + const response = await http.fetch(converted.request, converted.connection); + for (const [name, value] of response.headers) res.appendHeader(name, value); + res.writeHead(response.status).end(Buffer.from(await response.arrayBuffer())); +}); +own.on("upgrade", http.upgrade); +await new Promise((resolve) => own.listen(0, "127.0.0.1", resolve)); +const address = own.address(); +const port = typeof address === "object" && address !== null ? address.port : 0; + +// 3. Requests through the app's server get the same routes, checks and headers. +for (const path of ["/notes", "/nowhere"]) { + const response = await fetch(`http://127.0.0.1:${port}${path}`); + console.log(`GET ${path} -> ${response.status} ${await response.text()}`); + console.log(` x-content-type-options: ${response.headers.get("x-content-type-options")}`); +} + +// 4. What a Web `Request` cannot carry is refused at the conversion, as Chord's listener refuses it; the app's server +// lives on (`TRACE`: 501; a `Host` with userinfo: 403). +for (const [method, host] of [ + ["TRACE", `127.0.0.1:${port}`], + ["GET", `attacker@127.0.0.1:${port}`], +] as const) { + const status = await new Promise((resolve) => { + request({ host: "127.0.0.1", port, method, path: "/notes", headers: { host } }, (response) => { + response.resume(); + resolve(response.statusCode); + }).end(); + }); + console.log(`${method} /notes (Host: ${host}) -> ${status}`); +} + +// 5. The app closes its server and the component (it refuses, then waits for running requests), then the host. +await new Promise((resolve) => own.close(resolve)); +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts new file mode 100644 index 000000000..f08442bd0 --- /dev/null +++ b/packages/chord/test/next/examples/47-http-own-server/plugins/notes/server.ts @@ -0,0 +1,15 @@ +// The notes plugin's server facet: one route, the same whichever server runs it. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /notes": { + public: true, + handler: async (request) => ({ kind: "json", value: { notes: ["buy milk"], via: request.origin } }), + }, + }, + }), +); diff --git a/packages/chord/test/next/examples/48-http-baseline/main.ts b/packages/chord/test/next/examples/48-http-baseline/main.ts new file mode 100644 index 000000000..ee1483cd7 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/main.ts @@ -0,0 +1,88 @@ +// The security baseline (slice 10): on by default, the app decides only who someone is. Here the app signs people in +// with a signed cookie (`cookie`, `readCookie`) and the notes plugin contributes a page, routes and a webhook. Requests +// another site's page makes in a browser (Fetch Metadata says so) are never authenticated (so a route not marked +// public answers them 401: deny by default, slice 13), a foreign `Origin` is refused, but a link to a page still opens +// it signed in, and a cross-site webhook runs anonymous. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/48-http-baseline/main.ts +import { request } from "node:http"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import * as chordHttp from "../../../../src/next-node/http/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; + +type Principal = { readonly kind: "server" } | { readonly kind: "user"; readonly name: string }; +const secret = "a long random secret"; +const server = createHost({ principal: { kind: "server" }, onError: console.error }); +const http = chordHttp.createHttp(server, { listen: { host: "127.0.0.1", port: 0 }, onError: console.error }, context); + +// 1. The app's core facet: a sign-in route setting the cookie, and an authenticator reading it back. +const app = defineFacet((_host, self) => { + self.contribute(chordHttp.Http, { + routes: { + "POST /sign-in": { + public: true, + handler: async (request) => { + const set = chordHttp.cookie("user", String(await request.json()), { secret, request, sameSite: "Lax" }); + return { kind: "json", value: "signed in", headers: { "Set-Cookie": set } }; + }, + }, + }, + }); + self.contribute(chordHttp.Authenticators, (request) => { + const name = chordHttp.readCookie(request, "user", secret); + return name === undefined ? undefined : { kind: "user", name }; + }); +}); +const modules = { "@earendil-works/chord/next": chordNext, "@earendil-works/chord/next/http": chordHttp }; +const plugins = createPlugins( + { + roots: [fileURLToPath(new URL("./plugins/", import.meta.url))], + kinds: { server: { platform: "node", hostPackages: Object.keys(modules) } }, + onError: console.error, + }, + context, +); +const core = [ + { name: "chord.http", facet: http.facet }, + { name: "@example/app", facet: app }, +]; +await plugins.load(server, { kind: "server", core, modules }, context); +const { port } = await http.listening(context); +const base = `http://127.0.0.1:${port}`; + +// 2. Ada signs in; her browser keeps the cookie. +const signIn = await fetch(`${base}/sign-in`, { method: "POST", body: '"ada"' }); +const cookie = signIn.headers.get("set-cookie")!.split(";")[0]!; +console.log(`1. signed in: ${cookie.slice(0, 12)}…`); + +/** A request from Ada's browser: her cookie, and what Fetch Metadata says about who made it (`fetch` sets its own). */ +async function browser(label: string, method: string, path: string, headers: Record = {}, body = "") { + const answer = await new Promise((resolve) => { + const sent = request({ host: "127.0.0.1", port, method, path, headers: { cookie, ...headers } }, (response) => { + let text = ""; + response.on("data", (chunk) => { + text += chunk; + }); + response.on("end", () => resolve(`${response.statusCode} ${text}`)); + }); + sent.end(body); + }); + console.log(`${label}: ${answer}`); +} + +const ours = { "sec-fetch-site": "same-origin", origin: base }; +const evil = { "sec-fetch-site": "cross-site", origin: "https://evil.example" }; +await browser("2. our page reads the notes", "GET", "/notes", ours); +await browser("3. our page adds a note", "POST", "/notes", ours, '"call mom"'); +await browser("4. evil.example posts a note", "POST", "/notes", evil, '"pwned"'); +await browser("5. evil.example reads the notes", "GET", "/notes", { "sec-fetch-site": "cross-site" }); +const link = { "sec-fetch-site": "cross-site", "sec-fetch-mode": "navigate", "sec-fetch-dest": "document" }; +await browser("6. a link on evil.example to the page", "GET", "/", link); +await browser("7. a build server's webhook", "POST", "/hooks/build", { origin: "https://ci.example" }); + +await http.close(context); +await server.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts new file mode 100644 index 000000000..52b8c142f --- /dev/null +++ b/packages/chord/test/next/examples/48-http-baseline/plugins/notes/server.ts @@ -0,0 +1,33 @@ +// The notes plugin's server facet: the notes page (a link from another site may open it), the notes as JSON, adding +// a note, and a webhook another site calls (cross-site: it always runs anonymous, so it is public). Nothing else is +// public: a request nobody signed in gets 401 before this code runs (deny by default). + +import { defineFacet } from "@earendil-works/chord/next"; +import { Http } from "@earendil-works/chord/next/http"; + +const notes = ["buy milk"]; + +export default defineFacet((_host, self) => + self.contribute(Http, { + routes: { + "GET /": { + page: true, + handler: async (_request, ctx) => ({ kind: "html", body: `hi ${who(ctx.principal)}` }), + }, + "GET /notes": async (_request, ctx) => ({ kind: "json", value: { notes, as: who(ctx.principal) } }), + "POST /notes": async (request) => { + notes.push(String(await request.json())); + return { kind: "json", value: notes }; + }, + "POST /hooks/build": { + crossSite: true, + handler: async (_request, ctx) => ({ kind: "json", value: who(ctx.principal) }), + }, + }, + }), +); + +/** Who a principal is, for the answers. */ +function who(principal: unknown): string { + return principal === undefined ? "anonymous" : JSON.stringify(principal); +} diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v2.ts b/packages/chord/test/next/examples/49-plugin-api/app/v2.ts new file mode 100644 index 000000000..9b1da645e --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v2.ts @@ -0,0 +1,4 @@ +// Acme's API, version 2 (`@acme/app/v2`): still provided, as a thin layer over v3. Version 2 called the greetings +// registry `Hellos`; the layer keeps that name, so plugins written against v2 build and run unchanged. + +export { constant, defineFacet, Greetings as Hellos, local, str } from "./v3.ts"; diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3.ts new file mode 100644 index 000000000..149c4274a --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3.ts @@ -0,0 +1,23 @@ +// Acme's API, version 3, shared by every host kind (`@acme/app/v3`): what a plugin imports from its host. It re-exports +// the part of Chord Acme supports (plugins never import Chord directly) next to Acme's own part: the `Greetings` +// registry every kind has. Examples 49 to 52 build against it; a real app keeps it in `src/api/v3/`. + +import { defineRegistry } from "../../../../../src/next/index.ts"; + +export { + constant, + defineFacet, + defineRegistry, + defineState, + int, + local, + method, + obj, + str, +} from "../../../../../src/next/index.ts"; + +/** Lines of greeting, one per contributing plugin; the app's start-data facet serves it (composing them in order). */ +export const Greetings = defineRegistry("acme.greetings", { version: 1 }); + +/** The API's own version, for plugins that print it. */ +export const API_VERSION = "v3"; diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3/server.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3/server.ts new file mode 100644 index 000000000..445c8647c --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3/server.ts @@ -0,0 +1,6 @@ +// Acme's API v3 for server facets only (`@acme/app/v3/server`): what exists on the server and nowhere else. + +/** The server's name. */ +export function serverName(): string { + return "the acme server"; +} diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3/web.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3/web.ts new file mode 100644 index 000000000..23e38cb0a --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3/web.ts @@ -0,0 +1,6 @@ +// Acme's API v3 for web facets only (`@acme/app/v3/web`): the panels a page shows. + +import { defineRegistry } from "../../../../../../src/next/index.ts"; + +/** A panel's title, one per contributing plugin. */ +export const Panels = defineRegistry<{ readonly title: string }, readonly string[]>("acme.panels", { version: 1 }); diff --git a/packages/chord/test/next/examples/49-plugin-api/app/v3/worker.ts b/packages/chord/test/next/examples/49-plugin-api/app/v3/worker.ts new file mode 100644 index 000000000..aa51aae44 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/app/v3/worker.ts @@ -0,0 +1,6 @@ +// Acme's API v3 for worker facets only (`@acme/app/v3/worker`): a registry of the workers' indexers (example 52). + +import { defineRegistry } from "../../../../../../src/next/index.ts"; + +/** Word indexers a worker runs over a project's notes: each contributes its name. */ +export const Indexers = defineRegistry<{ readonly name: string }, readonly string[]>("acme.indexers", { version: 1 }); diff --git a/packages/chord/test/next/examples/49-plugin-api/main.ts b/packages/chord/test/next/examples/49-plugin-api/main.ts new file mode 100644 index 000000000..145cf1f93 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/main.ts @@ -0,0 +1,80 @@ +// The app's versioned API (slice 12 §1, decision "Plugins import only the app's versioned API"). Plugins import their +// host only through the app's API modules, a version in the path: `@acme/app/v3` (shared by every kind) and its +// per-kind modules (`@acme/app/v3/server`, `/v3/web`); the app lists the versions it still provides (`api.versions`) +// and each kind's modules (`kinds..hostPackages`). Here (`plugins/`, the API in `app/`): a v3 plugin and a v2 +// plugin both load (v2 is a thin layer over v3); refused at build, each with its error: a v1 plugin (no longer +// provided), a facet mixing v2 and v3, a direct import of Chord, a server facet importing the web's module. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/49-plugin-api/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../../src/next/host/index.ts"; +import { createPlugins, evaluateFacet } from "../../../../src/next-node/plugins/index.ts"; +import * as v2 from "./app/v2.ts"; +import * as v3server from "./app/v3/server.ts"; +import * as v3web from "./app/v3/web.ts"; +import * as v3 from "./app/v3.ts"; + +const context = BACKGROUND_CONTEXT; +const root = fileURLToPath(new URL("./plugins/", import.meta.url)); + +// 1. The component with the app's API: its name and versions (newest first), and per kind the modules its facets may +// import. Chord is no host package of plugins: the API re-exports what Acme supports of it. +const plugins = createPlugins( + { + roots: [{ path: root, required: false }], + api: { name: "@acme/app", versions: ["v3", "v2"] }, + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server", "@acme/app/v2"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web", "@acme/app/v2"] }, + }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. The server's modules (this process's instances) and its own start-data facet, base code: it serves the +// `Greetings` registry, composing the lines in plugin-name order. +const modules = { "@acme/app/v3": v3, "@acme/app/v3/server": v3server, "@acme/app/v2": v2 }; +const core = defineFacet((host) => host.serve(v3.Greetings, { compose: (table) => table.map(({ value }) => value) })); +const host = createHost({ onError: () => {} }); +const loaded = await plugins.load( + host, + { kind: "server", core: [{ name: "@acme/core", facet: core }], modules }, + context, +); + +// 3. Who loaded, and who was refused (each build error names the file, line and rule). +console.log("1. greetings:"); +for (const line of (await host.registry(v3.Greetings, context)).value) console.log(` ${line}`); +console.log("2. refused at build:"); +for (const { plugin, error } of loaded.failed) { + const [, ...errors] = (error as Error).message.split("\n"); + console.log(` ${plugin}: ${errors.join("; ")}`); +} + +// 4. The web kind provides `/v3/web`: the panel plugin's web facet builds, and a page (here a host in this process, +// given the web's modules) evaluates and loads it. +const built = await plugins.build(["@acme/panel"], context); +console.log("3. the panel plugin:", built.facets.map((facet) => `${facet.facet} ${facet.kind}`).join(", ")); +const web = built.facets.find((facet) => facet.facet === "web" && facet.kind === "script"); +if (web?.kind === "script") { + const webModules = { "@acme/app/v3": v3, "@acme/app/v3/web": v3web, "@acme/app/v2": v2 }; + const panels = defineFacet((h) => + h.serve(v3web.Panels, { compose: (table) => table.map(({ value }) => value.title) }), + ); + const page = createHost({ onError: () => {} }); + const facet = evaluateFacet(web, { hostPackages: webModules, plugin: built.plugins["@acme/panel"]! }); + await page.load( + [ + { name: "@acme/page", facet: panels }, + { name: "@acme/panel", facet }, + ], + context, + ); + console.log(" panels on the page:", (await page.registry(v3web.Panels, context)).value.join(", ")); + await page.close(context); +} + +await host.close(context); +plugins.close(); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/env.d.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/env.d.ts new file mode 100644 index 000000000..4b2598837 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/env.d.ts @@ -0,0 +1,9 @@ +// Types of the API version this plugin was written against: the app no longer provides it (no module to type-check +// against, the build refuses the import), so the plugin's last author left these. + +declare module "@acme/app/v1" { + export function defineFacet( + setup: (host: unknown, self: { contribute(registry: unknown, value: string): void }) => void, + ): unknown; + export const Greetings: unknown; +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/package.json new file mode 100644 index 000000000..fdfe3f61d --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/ancient", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/server.ts new file mode 100644 index 000000000..b1b42e926 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/ancient/server.ts @@ -0,0 +1,5 @@ +// A plugin written against v1, which the app no longer provides: refused at build, naming the versions it has. + +import { defineFacet, Greetings } from "@acme/app/v1"; + +export default defineFacet((_host, self) => self.contribute(Greetings, "hello from v1")); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/direct/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/package.json new file mode 100644 index 000000000..00bf91b51 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/direct", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/direct/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/server.ts new file mode 100644 index 000000000..13a290169 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/direct/server.ts @@ -0,0 +1,5 @@ +// A plugin importing Chord directly: refused, pointing to the app's API (which re-exports what plugins may use). + +import { defineFacet } from "@earendil-works/chord/next"; + +export default defineFacet(() => {}); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/hello/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/package.json new file mode 100644 index 000000000..46fb32844 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/hello", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/hello/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/server.ts new file mode 100644 index 000000000..2fee479bf --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/hello/server.ts @@ -0,0 +1,6 @@ +// A plugin written against v3: the shared module and the server's own module, one version for the whole facet. + +import { defineFacet, Greetings } from "@acme/app/v3"; +import { serverName } from "@acme/app/v3/server"; + +export default defineFacet((_host, self) => self.contribute(Greetings, `hello from a v3 plugin on ${serverName()}`)); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/package.json new file mode 100644 index 000000000..e35b0f9c6 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/legacy", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/server.ts new file mode 100644 index 000000000..360ac3775 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/legacy/server.ts @@ -0,0 +1,5 @@ +// A plugin written against v2 a year ago: the app still lists v2, so it builds and runs, through v2's names. + +import { defineFacet, Hellos } from "@acme/app/v2"; + +export default defineFacet((_host, self) => self.contribute(Hellos, "hello from a v2 plugin")); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/package.json new file mode 100644 index 000000000..922f1ac5c --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/mixed", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/server.ts new file mode 100644 index 000000000..7d84e0f69 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/mixed/server.ts @@ -0,0 +1,6 @@ +// A facet importing two versions of the API: refused (v2's and v3's registries may be different ones). + +import { Hellos } from "@acme/app/v2"; +import { defineFacet } from "@acme/app/v3"; + +export default defineFacet((_host, self) => self.contribute(Hellos, "hello from two versions")); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/panel/package.json b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/package.json new file mode 100644 index 000000000..c5fb2d9ab --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/panel", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/panel/server.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/server.ts new file mode 100644 index 000000000..d26507ccf --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/server.ts @@ -0,0 +1,7 @@ +// A server facet importing the web's module of the API: refused, naming the kind (a page's registry has no place on +// the server). + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: "a panel" })); diff --git a/packages/chord/test/next/examples/49-plugin-api/plugins/panel/web.ts b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/web.ts new file mode 100644 index 000000000..50abf6434 --- /dev/null +++ b/packages/chord/test/next/examples/49-plugin-api/plugins/panel/web.ts @@ -0,0 +1,6 @@ +// The panel plugin's web facet: the web's module of the API is for exactly this kind. + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: "word count" })); diff --git a/packages/chord/test/next/examples/50-plugin-imports/main.ts b/packages/chord/test/next/examples/50-plugin-imports/main.ts new file mode 100644 index 000000000..36b1b6af7 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/main.ts @@ -0,0 +1,96 @@ +// Imports between plugins (slice 12 §2, §3; decision "Contracts come from where the plugin's code came from"). A plugin +// carries the contract it was written against: plugins of one root (one repository, one commit) import each other's +// contract files by relative path, copied into the importer's script; a plugin of another repository takes them as a +// dependency pinned by an npm version or a git ref, inlined at the installed commit. Per-kind contract files +// (`contract.web.ts`) are for their kind only. Here: the app's root (`plugins/`, required) with notes, search (its server +// facet using notes' shared contract, its web facet notes' web file) and peek (a server facet reaching the web file +// through a helper: refused); the user's root (`user/`) with Bob's stats plugin, whose `@bob/words` dependency is pinned +// to the first commit of a local git repository (`words/` committed twice; no network), and reach (a relative path +// out of its root: refused). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/50-plugin-imports/main.ts +import { execFileSync } from "node:child_process"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { int, local, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins, PluginError } from "../../../../src/next-node/plugins/index.ts"; +import * as v3server from "../49-plugin-api/app/v3/server.ts"; +import * as v3 from "../49-plugin-api/app/v3.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example installs into a plugin: it works on a copy of its directories, removed at the end. +const copy = await mkdtemp(join(tmpdir(), "chord-example-50-")); +for (const dir of ["plugins", "user", "words"]) { + await cp(fileURLToPath(new URL(`./${dir}/`, import.meta.url)), join(copy, dir), { recursive: true }); +} + +// 1. `@bob/words` in its own git repository: version 1 committed, then version 2. Bob pinned the first commit. +const git = (cwd: string, ...args: string[]) => + execFileSync("git", ["-c", "user.email=bob@example.com", "-c", "user.name=bob", ...args], { cwd, encoding: "utf8" }); +const words = join(copy, "words"); +git(words, "init", "--quiet"); +git(words, "add", "--all"); +git(words, "commit", "--quiet", "--message", "words 1"); +const pinned = git(words, "rev-parse", "HEAD").trim(); +const source = await readFile(join(words, "words.ts"), "utf8"); +await writeFile(join(words, "words.ts"), source.replace('VERSION = "1"', 'VERSION = "2"')); +git(words, "commit", "--quiet", "--all", "--message", "words 2"); +git(copy, "clone", "--quiet", "--bare", words, join(copy, "words.git")); +const stats = join(copy, "user", "stats", "package.json"); +const manifest = JSON.parse(await readFile(stats, "utf8")) as { dependencies: Record }; +manifest.dependencies["@bob/words"] = `git+file://${join(copy, "words.git")}#${pinned}`; +await writeFile(stats, JSON.stringify(manifest, null, "\t")); +console.log("1. @bob/stats depends on", manifest.dependencies["@bob/words"].replace(copy, "…")); + +// 2. The component: the app's root required, the user's not; the app's API (example 49's) for both kinds. +const plugins = createPlugins( + { + roots: [{ path: join(copy, "plugins"), required: true }, join(copy, "user")], + api: { name: "@acme/app", versions: ["v3"] }, + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web"] }, + }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 3. Bob's dependencies, installed by npm into his plugin's node_modules (a git dependency at a ref is npm's own). +await plugins.installDependencies("@bob/stats", context); + +// 4. The server loads both roots. The peek plugin of the app's root fails: so the required root's load rejects (a +// `PluginError`, after the load settled: the others are loaded), and the app decides (here: it reports and goes on). +const modules = { "@acme/app/v3": v3, "@acme/app/v3/server": v3server }; +const host = createHost({ onError: () => {} }); +try { + await plugins.load(host, { kind: "server", core: [], modules }, context); +} catch (error) { + if (!(error instanceof PluginError)) throw error; + console.log("2. the load rejects:", error.name, "for", error.required.map(({ plugin }) => plugin).join(", ")); + for (const { plugin, error: cause } of error.failed) { + console.log(` ${plugin}: ${(cause as Error).message.split("\n").slice(1).join("; ")}`); + } +} +const Search = local("acme.search", { version: 1, params: {}, members: { hits: int() } }); +const Stats = local("bob.stats", { version: 1, params: {}, members: { text: str() } }); +console.log("3. search hits:", (await host.get(Search, {}, context)).hits.value); +console.log(" stats:", (await host.get(Stats, {}, context)).text.value); + +// 5. The search plugin's builds: what each copied from the notes plugin (its server facet the shared file, its web +// facet the web file too). +const built = await plugins.build(["@acme/search"], context); +for (const facet of built.facets) { + if (facet.kind !== "script") continue; + const copied = facet.copied.map((file) => file.slice(copy.length + 1)); + console.log(`4. @acme/search ${facet.facet} copied: ${copied.join(", ")}`); +} + +await host.close(context); +plugins.close(); +await rm(copy, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.ts new file mode 100644 index 000000000..d7a17cf0d --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.ts @@ -0,0 +1,7 @@ +// The notes plugin's shared contract: data only, importable from every kind. Other plugins of this root import it by +// relative path; the build copies it into their scripts. + +import { local, str } from "@acme/app/v3"; + +/** The notes, as one text. */ +export const Notes = local("acme.notes", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.web.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.web.ts new file mode 100644 index 000000000..f5f6806b9 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/contract.web.ts @@ -0,0 +1,5 @@ +// The notes plugin's web contract file (a file of the web kind: `.web.` before its extension): only web facets may +// import it, directly or through other files. + +/** The panel the notes plugin offers pages that list notes. */ +export const notesPanel = { title: "notes" } as const; diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/package.json b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/package.json new file mode 100644 index 000000000..1c76a10b8 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/server.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/server.ts new file mode 100644 index 000000000..4f4eafc17 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/notes/server.ts @@ -0,0 +1,6 @@ +// The notes plugin's server facet: serves `Notes`. + +import { constant, defineFacet } from "@acme/app/v3"; +import { Notes } from "./contract.ts"; + +export default defineFacet((host) => host.serve(Notes, () => ({ text: constant("buy milk, call bob, ship it") }))); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/package.json b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/package.json new file mode 100644 index 000000000..14832d7dd --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/peek", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/server.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/server.ts new file mode 100644 index 000000000..dae94d9d7 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/server.ts @@ -0,0 +1,6 @@ +// The peek plugin's server facet reaches a web file through its helper: refused at build, naming the chain. + +import { defineFacet } from "@acme/app/v3"; +import { notesPanel } from "./view.ts"; + +export default defineFacet(() => console.log(notesPanel)); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/view.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/view.ts new file mode 100644 index 000000000..d984e1271 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/peek/view.ts @@ -0,0 +1,3 @@ +// A helper of the peek plugin that re-exports the notes plugin's web contract file. + +export { notesPanel } from "../notes/contract.web.ts"; diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/search/package.json b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/package.json new file mode 100644 index 000000000..6f7be6d36 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/search", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/search/server.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/server.ts new file mode 100644 index 000000000..5b28876fd --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/server.ts @@ -0,0 +1,14 @@ +// The search plugin's server facet: uses the notes plugin's contract, imported by relative path (same root, same +// commit: a copy of that file in this script). + +import { defineFacet, int, local } from "@acme/app/v3"; +import { Notes } from "../notes/contract.ts"; + +const Search = local("acme.search", { version: 1, params: {}, members: { hits: int() } }); + +export default defineFacet((host) => + host.serve(Search, (_params, self) => { + const notes = self.get(Notes, {}); + return { hits: self.derive([notes.text], ([text]) => text.split(",").length) }; + }), +); diff --git a/packages/chord/test/next/examples/50-plugin-imports/plugins/search/web.ts b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/web.ts new file mode 100644 index 000000000..7f000cfdb --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/plugins/search/web.ts @@ -0,0 +1,7 @@ +// The search plugin's web facet: imports the notes plugin's web contract file, which web facets may. + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; +import { notesPanel } from "../notes/contract.web.ts"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: `search ${notesPanel.title}` })); diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/reach/package.json b/packages/chord/test/next/examples/50-plugin-imports/user/reach/package.json new file mode 100644 index 000000000..d64b400a3 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/reach/package.json @@ -0,0 +1,5 @@ +{ + "name": "@bob/reach", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/reach/server.ts b/packages/chord/test/next/examples/50-plugin-imports/user/reach/server.ts new file mode 100644 index 000000000..3606563ba --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/reach/server.ts @@ -0,0 +1,7 @@ +// A user's plugin (another root) importing the app's notes contract by its path: refused, a relative path leaving its +// root; a plugin of another repository takes contracts as a dependency (or from the app's API). + +import { defineFacet } from "@acme/app/v3"; +import { Notes } from "../../plugins/notes/contract.ts"; + +export default defineFacet(() => console.log(Notes.name)); diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/stats/env.d.ts b/packages/chord/test/next/examples/50-plugin-imports/user/stats/env.d.ts new file mode 100644 index 000000000..4aca7b708 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/stats/env.d.ts @@ -0,0 +1,6 @@ +// Types of the stats plugin's dependency before it is installed (its own types once installed). + +declare module "@bob/words" { + export const VERSION: string; + export function words(text: string): number; +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/stats/package.json b/packages/chord/test/next/examples/50-plugin-imports/user/stats/package.json new file mode 100644 index 000000000..46aee125f --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/stats/package.json @@ -0,0 +1,8 @@ +{ + "name": "@bob/stats", + "private": true, + "type": "module", + "dependencies": { + "@bob/words": "git+file:///path/to/words.git#" + } +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/user/stats/server.ts b/packages/chord/test/next/examples/50-plugin-imports/user/stats/server.ts new file mode 100644 index 000000000..6d6982abe --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/user/stats/server.ts @@ -0,0 +1,11 @@ +// Bob's stats plugin (the user's root): counts words with `@bob/words`, a dependency pinned to a git commit. The build +// inlines the package at the installed commit, whatever the repository holds since. + +import { constant, defineFacet, local, str } from "@acme/app/v3"; +import { VERSION, words } from "@bob/words"; + +const Stats = local("bob.stats", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => + host.serve(Stats, () => ({ text: constant(`${words("buy milk, call bob")} words (@bob/words ${VERSION})`) })), +); diff --git a/packages/chord/test/next/examples/50-plugin-imports/words/package.json b/packages/chord/test/next/examples/50-plugin-imports/words/package.json new file mode 100644 index 000000000..5fee19049 --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/words/package.json @@ -0,0 +1,6 @@ +{ + "name": "@bob/words", + "version": "1.0.0", + "type": "module", + "exports": { ".": "./words.ts" } +} diff --git a/packages/chord/test/next/examples/50-plugin-imports/words/words.ts b/packages/chord/test/next/examples/50-plugin-imports/words/words.ts new file mode 100644 index 000000000..de1226fed --- /dev/null +++ b/packages/chord/test/next/examples/50-plugin-imports/words/words.ts @@ -0,0 +1,9 @@ +// `@bob/words`: a package in its own git repository (the example commits it twice: this file, then a version 2). + +/** The version of this commit. */ +export const VERSION = "1"; + +/** The words in `text`. */ +export function words(text: string): number { + return text.split(/\s+/).filter(Boolean).length; +} diff --git a/packages/chord/test/next/examples/51-single-file-plugin/main.ts b/packages/chord/test/next/examples/51-single-file-plugin/main.ts new file mode 100644 index 000000000..3003288e9 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/main.ts @@ -0,0 +1,88 @@ +// Single-file plugins (slice 12 §4, decision "Single-file plugins"). A file at a root's top named +// `..ts(x)` is a facet of the plugin ``; files with one first name are one plugin. It has no +// `package.json`, so no dependencies: it imports the app's API only, defines its contracts inline, serves objects and +// contributes to registries. When it needs more, it becomes a directory with the same name, which a reload treats as +// the same plugin. Here (`plugins/`): `hello` (a server and a web file), `clock`, and `bad`, which imports another file +// and is refused; then `hello` becomes a directory with a helper file: the watcher marks the root, a build and a +// reload change the plugin `hello` in place. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/51-single-file-plugin/main.ts +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { local, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import * as v3server from "../49-plugin-api/app/v3/server.ts"; +import * as v3 from "../49-plugin-api/app/v3.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example turns a file into a directory: it works on a copy of `plugins/`, removed at the end. +const root = await mkdtemp(join(tmpdir(), "chord-example-51-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +// 1. The component, the server loaded: discovery found three single-file plugins; `bad` fails its build. +const plugins = createPlugins( + { + roots: [root], + api: { name: "@acme/app", versions: ["v3"] }, + kinds: { + server: { platform: "node", hostPackages: ["@acme/app/v3", "@acme/app/v3/server"] }, + web: { platform: "browser", hostPackages: ["@acme/app/v3", "@acme/app/v3/web"] }, + }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); +const modules = { "@acme/app/v3": v3, "@acme/app/v3/server": v3server }; +const host = createHost({ onError: () => {} }); +const loaded = await plugins.load(host, { kind: "server", core: [], modules }, context); +for (const plugin of plugins.discovery.plugins) { + const files = Object.values(plugin.facets).map((file) => file.slice(root.length + 1)); + console.log(`1. ${plugin.name} (${plugin.layout}): ${files.join(", ")}`); +} +for (const { plugin, error } of loaded.failed) { + console.log(` ${plugin} refused: ${(error as Error).message.split("\n").slice(1).join("; ")}`); +} +const Hello = local("example.hello", { version: 1, params: {}, members: { text: str() } }); +const Clock = local("example.clock", { version: 1, params: {}, members: { text: str() } }); +const hello = await host.get(Hello, {}, context); +console.log("2. hello:", hello.text.value, "| clock:", (await host.get(Clock, {}, context)).text.value); + +// 2. `hello` needs a second file: it becomes the directory `hello/` (a package.json named `hello`), its files removed. +await mkdir(join(root, "hello")); +await writeFile(join(root, "hello", "package.json"), JSON.stringify({ name: "hello", type: "module" })); +await writeFile(join(root, "hello", "greeting.ts"), 'export const greeting = "hello from a directory";\n'); +await writeFile( + join(root, "hello", "server.ts"), + [ + 'import { constant, defineFacet, local, str } from "@acme/app/v3";', + 'import { greeting } from "./greeting.ts";', + 'const Hello = local("example.hello", { version: 1, params: {}, members: { text: str() } });', + "export default defineFacet((host) => host.serve(Hello, () => ({ text: constant(greeting) })));", + ].join("\n"), +); +await rm(join(root, "hello.server.ts")); +await rm(join(root, "hello.web.ts")); +await until(() => plugins.state.value.set && plugins.state.value.dirty.includes("hello")); +console.log("3. marks:", JSON.stringify(plugins.state.value)); + +// 3. A build discovers again; the reload changes `hello` in place (the same plugin, its object swapped), a holder +// sees the new text. +const built = await plugins.build(plugins.state.value.dirty, context); +const report = await plugins.reload(host, built, context); +console.log("4. reload:", JSON.stringify(report.plugins), "| hello:", hello.text.value); +console.log(" hello is now:", plugins.discovery.plugins.find(({ name }) => name === "hello")?.layout); + +hello.release(); +await host.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/bad.server.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/bad.server.ts new file mode 100644 index 000000000..82f4fb859 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/bad.server.ts @@ -0,0 +1,6 @@ +// A single-file plugin importing another file: refused at build, saying to make it a directory. + +import { defineFacet } from "@acme/app/v3"; +import { helper } from "./helper.ts"; + +export default defineFacet(() => console.log(helper)); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/clock.server.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/clock.server.ts new file mode 100644 index 000000000..49ec01059 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/clock.server.ts @@ -0,0 +1,8 @@ +// Another single-file plugin: a line naming the server, through the server's module of the API. + +import { constant, defineFacet, local, str } from "@acme/app/v3"; +import { serverName } from "@acme/app/v3/server"; + +const Clock = local("example.clock", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Clock, () => ({ text: constant(`ticking on ${serverName()}`) }))); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.server.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.server.ts new file mode 100644 index 000000000..79f4cffc1 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.server.ts @@ -0,0 +1,8 @@ +// A single-file plugin named `hello` (its file name: `..ts`), its server facet: it defines its contract +// inline and serves it. No directory, no package.json: it imports only the app's API. + +import { constant, defineFacet, local, str } from "@acme/app/v3"; + +const Hello = local("example.hello", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Hello, () => ({ text: constant("hello from one file") }))); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.web.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.web.ts new file mode 100644 index 000000000..ad2790e7f --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/hello.web.ts @@ -0,0 +1,6 @@ +// The same plugin's web facet: a second file with the same first name. + +import { defineFacet } from "@acme/app/v3"; +import { Panels } from "@acme/app/v3/web"; + +export default defineFacet((_host, self) => self.contribute(Panels, { title: "hello" })); diff --git a/packages/chord/test/next/examples/51-single-file-plugin/plugins/helper.ts b/packages/chord/test/next/examples/51-single-file-plugin/plugins/helper.ts new file mode 100644 index 000000000..13e787649 --- /dev/null +++ b/packages/chord/test/next/examples/51-single-file-plugin/plugins/helper.ts @@ -0,0 +1,3 @@ +// A plain file at the root's top: not a plugin (its name has no kind), and no single-file plugin may import it. + +export const helper = "a helper"; diff --git a/packages/chord/test/next/examples/52-test-plugin/main.ts b/packages/chord/test/next/examples/52-test-plugin/main.ts new file mode 100644 index 000000000..e5c5dfc9c --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/main.ts @@ -0,0 +1,68 @@ +// Testing a plugin alone (slice 12 §6, decision "Testing a plugin alone"). `testPlugin(dir, options, ctx)` builds the +// plugin as production does (the same build, the app's API checked), evaluates it with the app's real API modules and +// loads it on an in-memory host (a `:memory:` store) beside stubs: ordinary facets serving the contracts it uses with +// fake data. Registries it contributes to that nobody serves are served by the helper, which collects the entries; +// contracts its requests wait for unserved are listed. Here (`plugins/`): the stats plugin, with a stub for the notes +// plugin, its indexer read from the collected `Indexers`, and the `Tags` contract it waits for listed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/52-test-plugin/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { testPlugin } from "../../../../src/next-node/plugins/index.ts"; +import * as v3worker from "../49-plugin-api/app/v3/worker.ts"; +import * as v3 from "../49-plugin-api/app/v3.ts"; + +const context = BACKGROUND_CONTEXT; + +// The contracts the test uses, declared again with the API's builders (contracts are data: the same name, version and +// schema bind). The plugins' own files import `@acme/app/v3`, a host package this process has no module path for. +const Notes = v3.local("acme.notes", { version: 1, params: { project: v3.str() }, members: { text: v3.str() } }); +const Stats = v3.local("acme.stats", { version: 1, params: { project: v3.str() }, members: { words: v3.int() } }); +const TagLine = v3.local("acme.tag-line", { version: 1, params: { project: v3.str() }, members: { text: v3.str() } }); + +/** The worker host's principal type, and a user's. */ +type Principal = { readonly kind: "worker"; readonly project: string } | { readonly kind: "user"; readonly id: string }; + +// 1. A stub of the notes plugin: an ordinary facet serving `Notes` with a fixed text for every project. +const notesStub = v3.defineFacet((host) => + host.serve(Notes, ({ project }) => ({ text: v3.constant(`${project}: buy milk, call bob, ship it`) })), +); + +// 2. The plugin on a worker host of its own, acting as the worker's principal; the app's API as production has it. +const t = await testPlugin( + fileURLToPath(new URL("./plugins/stats/", import.meta.url)), + { + kind: "worker", + kinds: ["server", "worker", "web"], + api: { name: "@acme/app", versions: ["v3"] }, + modules: { "@acme/app/v3": v3, "@acme/app/v3/worker": v3worker }, + stubs: [{ name: "@acme/notes", facet: notesStub }], + principal: { kind: "worker", project: "p1" }, + }, + context, +); + +// 3. Its objects are live: get them, read their state. +const stats = await t.host.get(Stats, { project: "p1" }, context); +console.log("1. words in p1:", stats.words.value); + +// 4. What it contributed to `Indexers`, a registry of the app no stub serves: collected by the helper. +console.log( + "2. indexers:", + t.contributions(v3worker.Indexers).map(({ plugin, value }) => `${value.name} (${plugin})`), +); + +// 5. A contract nobody serves: its requests wait, and the helper names it, so the test knows what to stub next. +const tags = t.host.hold(TagLine, { project: "p1" }, context); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("3. waiting for:", t.unserved(), "| tag line:", tags.state.value.kind); + +// 6. Code outside objects as another principal (a user), and as the host's own. +const who = (principal: Principal | undefined) => (principal === undefined ? "nobody" : JSON.stringify(principal)); +console.log("4. root as the host:", who(await t.root(async (ctx) => ctx.principal, context))); +const user = { kind: "user", id: "u1" } as const; +console.log(" root as a user:", who(await t.root(async (ctx) => ctx.principal, context, { principal: user }))); + +stats.release(); +tags.release(); +await t.close(context); diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/notes/contract.ts b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/contract.ts new file mode 100644 index 000000000..964c8de65 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/contract.ts @@ -0,0 +1,6 @@ +// The notes plugin's contract (its facets are not part of this example: a test of the stats plugin stubs them). + +import { local, str } from "@acme/app/v3"; + +/** A project's notes, as one text. */ +export const Notes = local("acme.notes", { version: 1, params: { project: str() }, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/notes/package.json b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/package.json new file mode 100644 index 000000000..1c76a10b8 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/stats/package.json b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/package.json new file mode 100644 index 000000000..173f09134 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/package.json @@ -0,0 +1,5 @@ +{ + "name": "@acme/stats", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/52-test-plugin/plugins/stats/worker.ts b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/worker.ts new file mode 100644 index 000000000..a3c88b119 --- /dev/null +++ b/packages/chord/test/next/examples/52-test-plugin/plugins/stats/worker.ts @@ -0,0 +1,27 @@ +// The stats plugin's worker facet: words per project over the notes plugin's `Notes`, an indexer contributed to the +// workers' `Indexers` registry, and tags over a `Tags` contract (the tags plugin's; this example's test stubs none). + +import { defineFacet, int, local, str } from "@acme/app/v3"; +import { Indexers } from "@acme/app/v3/worker"; +import { Notes } from "../notes/contract.ts"; + +/** Words per project. */ +const Stats = local("acme.stats", { version: 1, params: { project: str() }, members: { words: int() } }); + +/** The tags plugin's contract, as this plugin uses it. */ +const Tags = local("acme.tags", { version: 1, params: { project: str() }, members: { list: str() } }); + +/** The tags of a project, as this plugin shows them. */ +const TagLine = local("acme.tag-line", { version: 1, params: { project: str() }, members: { text: str() } }); + +export default defineFacet((host, self) => { + self.contribute(Indexers, { name: "word count" }); + host.serve(Stats, ({ project }, self) => { + const notes = self.get(Notes, { project }); + return { words: self.derive([notes.text], ([text]) => text.split(/\s+/).filter(Boolean).length) }; + }); + host.serve(TagLine, ({ project }, self) => { + const tags = self.get(Tags, { project }); + return { text: self.derive([tags.list], ([list]) => `tags: ${list}`) }; + }); +}); diff --git a/packages/chord/test/next/examples/53-app/app/principals.ts b/packages/chord/test/next/examples/53-app/app/principals.ts new file mode 100644 index 000000000..cde89e19a --- /dev/null +++ b/packages/chord/test/next/examples/53-app/app/principals.ts @@ -0,0 +1,8 @@ +// The notes app's principals: its server and workers, the kit's owner, and the users its plugins may sign in. + +/** Who someone is in the notes app. */ +export type NotesPrincipal = + | { readonly kind: "server" } + | { readonly kind: "owner" } + | { readonly kind: "worker"; readonly project: string } + | { readonly kind: "user"; readonly id: string; readonly name: string }; diff --git a/packages/chord/test/next/examples/53-app/app/v1.ts b/packages/chord/test/next/examples/53-app/app/v1.ts new file mode 100644 index 000000000..45cb45764 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/app/v1.ts @@ -0,0 +1,31 @@ +// The notes app's API, version 1, shared by every host kind (`@notes/app/v1`): what its plugins import from their +// host. The part of Chord the app supports, the kit's start-data objects and files, the app's own host kind (a +// project, one worker per project) and its registry of home-page sections. The API only declares the registry: its +// provider is the home plugin of the app's required root, not base code. Examples 53 to 55 run this app; a real app +// keeps it in `src/api/v1.ts`. + +import { defineRegistry, hostKind, str } from "../../../../../src/next/index.ts"; + +export { + arr, + constant, + defineFacet, + defineRegistry, + defineState, + defineTable, + int, + local, + method, + obj, + remote, + str, +} from "../../../../../src/next/index.ts"; +export { AppInfo, Files, WorkerInfo } from "../../../../../src/next-node/app/index.ts"; + +/** A project of the app: its own worker process, store and files. */ +export const Project = hostKind("project", { id: str() }); + +/** The home page's sections, one per contributing plugin (title and the path it links to). */ +export const Sections = defineRegistry<{ readonly title: string; readonly path: string }, string[]>("notes.sections", { + version: 1, +}); diff --git a/packages/chord/test/next/examples/53-app/app/v1/server.ts b/packages/chord/test/next/examples/53-app/app/v1/server.ts new file mode 100644 index 000000000..de7f5a42b --- /dev/null +++ b/packages/chord/test/next/examples/53-app/app/v1/server.ts @@ -0,0 +1,5 @@ +// The notes app's API v1 for server facets only (`@notes/app/v1/server`): the HTTP registry plugins contribute routes +// to, and the kit's sessions. + +export { Sessions } from "../../../../../../src/next-node/app/index.ts"; +export { Http } from "../../../../../../src/next-node/http/index.ts"; diff --git a/packages/chord/test/next/examples/53-app/client.ts b/packages/chord/test/next/examples/53-app/client.ts new file mode 100644 index 000000000..2c26b82d6 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/client.ts @@ -0,0 +1,44 @@ +// A small HTTP client for examples 53 to 55, standing in for a browser and a command-line client: a request to the +// app on loopback, its status, `Set-Cookie` and JSON body. + +import { request } from "node:http"; + +/** An answer: status, the cookie it set (name=value), JSON body (or text). */ +export type Answer = { readonly status: number; readonly cookie: string | undefined; readonly body: unknown }; + +/** `method path` on the app at `port`, with `headers` and a JSON `body`. */ +export function call( + port: number, + method: string, + path: string, + headers: Readonly> = {}, + body?: unknown, +): Promise { + return new Promise((resolve, reject) => { + const sent = request( + { + host: "127.0.0.1", + port, + method, + path, + headers: { ...headers, ...(body === undefined ? {} : { "content-type": "application/json" }) }, + }, + (response) => { + const chunks: Buffer[] = []; + response.on("data", (chunk: Buffer) => chunks.push(chunk)); + response.on("end", () => { + const text = Buffer.concat(chunks).toString(); + const type = response.headers["content-type"] ?? ""; + const cookie = response.headers["set-cookie"]?.[0]?.split(";")[0]; + resolve({ + status: response.statusCode ?? 0, + cookie, + body: type.includes("json") ? JSON.parse(text) : text, + }); + }); + }, + ); + sent.on("error", reject); + sent.end(body === undefined ? undefined : JSON.stringify(body)); + }); +} diff --git a/packages/chord/test/next/examples/53-app/main.ts b/packages/chord/test/next/examples/53-app/main.ts new file mode 100644 index 000000000..42c2410df --- /dev/null +++ b/packages/chord/test/next/examples/53-app/main.ts @@ -0,0 +1,56 @@ +// The app kit (slice 13): an app is its API (`app/`), its built-in plugins (`plugins/`: home and notes) and a start +// script (`start.ts`) calling `startServer`; its workers' entry (`worker.ts`) calls `startWorker`. The kit makes the data +// directory, the server's store and host, HTTP (deny by default: no session, 401), one worker per project with its own +// store and files, and the owner's sign-in: a one-time owner link for the browser, an owner token file for command-line +// clients on the same machine. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/53-app/main.ts +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { call } from "./client.ts"; +import { startNotes } from "./start.ts"; + +const dataDir = join(await mkdtemp(join(tmpdir(), "notes-")), "data"); + +// 1. Start the app. The start script prints the owner link (single use, 10 minutes, a new one at each start). +const app = await startNotes(dataDir, context); +const link = new URL(app.ownerLink()); +const port = Number(link.port); +console.log("1. owner link:", `${link.origin}${link.pathname}#…`); +console.log(" data directory:", (await readdir(dataDir)).sort().join(", ")); + +// 2. Nobody is signed in: every plugin route answers 401 before the plugin runs. +console.log("2. anonymous GET /:", (await call(port, "GET", "/")).status); + +// 3. The owner opens the link: the page posts the token from the URL's fragment (never sent in a URL, so never in a +// log) and gets the session cookie. The link is used up. +const page = await call(port, "GET", link.pathname); +console.log( + "3. the link's page:", + page.status, + String(page.body).includes("location.hash") ? "(reads the fragment)" : "", +); +const signedIn = await call(port, "POST", link.pathname, {}, { token: link.hash.slice(1) }); +const browser = { cookie: signedIn.cookie ?? "" }; +console.log(" signed in:", signedIn.status, signedIn.body); +console.log(" the link again:", (await call(port, "POST", link.pathname, {}, { token: link.hash.slice(1) })).status); + +// 4. Signed in, the home plugin reads the kit's `AppInfo` and lists the sections of its registry (the notes plugin's); +// the notes plugin's routes reach the project's worker (it starts on the first request) with its store and the +// start data the app gave it. +console.log("4. GET /:", JSON.stringify((await call(port, "GET", "/", browser)).body)); +await call(port, "POST", "/notes/p1", browser, "milk, eggs"); +console.log(" GET /notes/p1:", JSON.stringify((await call(port, "GET", "/notes/p1", browser)).body)); +console.log(" worker directories:", (await readdir(join(dataDir, "workers", "project"))).join(", ")); + +// 5. A command-line client on the same machine reads the owner token file (mode 0600) and sends it as a bearer token. +const token = (await readFile(app.ownerTokenFile, "utf8")).trim(); +const cli = { authorization: `Bearer ${token}` }; +console.log("5. CLI GET /:", JSON.stringify((await call(port, "GET", "/", cli)).body)); + +// 6. Close: HTTP, workers, host, store, in that order. +await app.close(context); +await rm(join(dataDir, ".."), { recursive: true, force: true }); +console.log("6. closed"); diff --git a/packages/chord/test/next/examples/53-app/plugins/home/package.json b/packages/chord/test/next/examples/53-app/plugins/home/package.json new file mode 100644 index 000000000..c64b0510f --- /dev/null +++ b/packages/chord/test/next/examples/53-app/plugins/home/package.json @@ -0,0 +1,5 @@ +{ + "name": "@notes/home", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/53-app/plugins/home/server.ts b/packages/chord/test/next/examples/53-app/plugins/home/server.ts new file mode 100644 index 000000000..608e869b8 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/plugins/home/server.ts @@ -0,0 +1,27 @@ +// The home plugin's server facet: it serves the app's `Sections` registry (the provider of a core registry is a plugin +// of the app's required root, so it reloads like any plugin) and an object reading it, and `GET /` says who is signed +// in to which app (the kit's `AppInfo`, the start data every server plugin can read) with the sections plugins +// contributed. + +import { AppInfo, arr, defineFacet, local, Sections, str } from "@notes/app/v1"; +import { Http } from "@notes/app/v1/server"; + +/** The home page's data: the composed sections. */ +const Home = local("notes.home", { version: 1, params: {}, members: { sections: arr(str()) } }); + +export default defineFacet((host, self) => { + host.serve(Sections, { compose: (table) => table.map(({ value }) => `${value.title}: ${value.path}`) }); + host.serve(Home, (_params, home) => ({ + sections: home.derive([home.registry(Sections)], ([sections]) => sections), + })); + self.contribute(Http, { + routes: { + "GET /": async (_request, ctx) => { + const info = await ctx.get(AppInfo, {}); + const page = await ctx.get(Home, {}); + const app = `${info.name.value} ${info.version.value}`; + return { kind: "json", value: { app, you: ctx.principal ?? null, sections: page.sections.value } }; + }, + }, + }); +}); diff --git a/packages/chord/test/next/examples/53-app/plugins/notes/contract.ts b/packages/chord/test/next/examples/53-app/plugins/notes/contract.ts new file mode 100644 index 000000000..3304ba83f --- /dev/null +++ b/packages/chord/test/next/examples/53-app/plugins/notes/contract.ts @@ -0,0 +1,22 @@ +// A project's notes: served by the project's worker (`Project({ id })`), reached from the server by id. + +import { method, obj, Project, remote, str } from "@notes/app/v1"; + +export const Notes = remote("notes.notes", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Project({ id }), + variants: { + main: { + state: { text: str(), start: str() }, + methods: { + /** Replaces the project's text. */ + write: method(str()), + /** Stores an attachment in the project's files; its id. */ + attach: method(obj({ name: str(), body: str() }), str()), + /** An attachment's text. */ + attachment: method(str(), str()), + }, + }, + }, +}); diff --git a/packages/chord/test/next/examples/53-app/plugins/notes/package.json b/packages/chord/test/next/examples/53-app/plugins/notes/package.json new file mode 100644 index 000000000..c345893f7 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@notes/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/53-app/plugins/notes/server.ts b/packages/chord/test/next/examples/53-app/plugins/notes/server.ts new file mode 100644 index 000000000..535c60157 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/plugins/notes/server.ts @@ -0,0 +1,23 @@ +// The notes plugin's server facet: routes reading and writing a project's notes, and its section of the home page. +// No route is public: the kit refuses anyone without a session (401) before they run. + +import { defineFacet, Sections } from "@notes/app/v1"; +import { Http } from "@notes/app/v1/server"; +import { Notes } from "./contract.ts"; + +export default defineFacet((_host, self) => { + self.contribute(Sections, { title: "Notes", path: "/notes/:id" }); + self.contribute(Http, { + routes: { + "GET /notes/:id": async (request, ctx) => { + const notes = await ctx.get(Notes, { id: request.params.id ?? "" }); + return { kind: "json", value: { text: notes.text.value, start: notes.start.value } }; + }, + "POST /notes/:id": async (request, ctx) => { + const notes = await ctx.get(Notes, { id: request.params.id ?? "" }); + await notes.write(String(await request.json()), ctx); + return { kind: "json", value: "saved" }; + }, + }, + }); +}); diff --git a/packages/chord/test/next/examples/53-app/plugins/notes/worker.ts b/packages/chord/test/next/examples/53-app/plugins/notes/worker.ts new file mode 100644 index 000000000..f034c7171 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/plugins/notes/worker.ts @@ -0,0 +1,49 @@ +// The notes plugin's worker facet: the project's text in a persisted document (kept by a local object with empty +// params, the worker's one copy: a persisted state needs a stable identity), its attachments in the project's files +// (the kit's `Files`), and the start data the app gave the worker (`WorkerInfo`). + +import { defineFacet, defineState, Files, local, method, obj, str, WorkerInfo } from "@notes/app/v1"; +import { Notes } from "./contract.ts"; + +const Text = defineState("notes.text", { + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "" }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 20, +}); + +/** The project's text, one per worker. */ +const Store = local("notes.store", { version: 1, params: {}, members: { text: str(), write: method(str()) } }); + +export default defineFacet((host) => { + host.serve(Store, (_params, self) => { + const text = self.state(Text); + return { + text: self.derive([text], ([value]) => value.text), + write: async (next, ctx) => { + await text.change((draft) => { + draft.text = next; + }, ctx); + return null; + }, + }; + }); + host.serve(Notes, (_params, _principal, self) => { + const store = self.get(Store, {}); + const info = self.get(WorkerInfo, {}); + const files = self.get(Files, {}); + return { + variant: "main", + members: { + text: self.derive([store.text], ([text]) => text), + start: self.derive([info.start], ([start]) => JSON.stringify(start)), + write: (next, ctx) => store.write(next, ctx), + attach: ({ name, body }, ctx) => + files.put({ name, mime: "text/plain", bytes: new TextEncoder().encode(body) }, ctx), + attachment: async (id, ctx) => + new TextDecoder().decode(await files.read({ id, offset: 0, length: 65536 }, ctx)), + }, + }; + }); +}); diff --git a/packages/chord/test/next/examples/53-app/start.ts b/packages/chord/test/next/examples/53-app/start.ts new file mode 100644 index 000000000..325152da0 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/start.ts @@ -0,0 +1,50 @@ +// The notes app's start: everything the app itself decides, given to `startServer`. The kit does the rest: the data +// directory, the store, the host, HTTP with sessions and the owner link, a workers component for projects, and the +// plugins (`plugins/` here, the app's required root, and `/plugins/`, the user's). +// Examples 53 to 55 call `startNotes`; a real app's `start.ts` calls it with `process.argv[2]` and prints the link. + +import { fileURLToPath } from "node:url"; +import type { Context } from "../../../../src/context/index.ts"; +import { type App, startServer } from "../../../../src/next-node/app/index.ts"; +import type { NotesPrincipal } from "./app/principals.ts"; +import * as v1server from "./app/v1/server.ts"; +import * as v1 from "./app/v1.ts"; + +/** The app's API: its name, versions (newest first) and this process's modules (the worker's entry has the same). */ +export const API = { + name: "@notes/app", + versions: ["v1"], + modules: { "@notes/app/v1": v1, "@notes/app/v1/server": v1server }, +}; + +/** The app's facet kinds. */ +export const KINDS = { server: { platform: "node" }, worker: { platform: "node" } } as const; + +/** Starts the notes app on a free loopback port with its data in `dataDir`. */ +export function startNotes(dataDir: string, context: Context): Promise> { + return startServer( + { + name: "notes", + dataDir, + root: fileURLToPath(new URL("./plugins/", import.meta.url)), + listen: { host: "127.0.0.1", port: 0 }, + api: API, + kinds: KINDS, + principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", project: address.key }) }, + workers: [ + { + kind: v1.Project, + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + // What each project's worker gets as `WorkerInfo.start`. + start: (address) => ({ project: address.key }), + // Who may reach a project: the owner (through the server's routes) and the server itself. + authorize: (principal) => principal.kind === "owner" || principal.kind === "server", + idleMs: () => 60_000, + }, + ], + onError: (error) => console.error("app:", error.kind, error.error), + }, + context, + ); +} diff --git a/packages/chord/test/next/examples/53-app/worker.ts b/packages/chord/test/next/examples/53-app/worker.ts new file mode 100644 index 000000000..bffa65fd2 --- /dev/null +++ b/packages/chord/test/next/examples/53-app/worker.ts @@ -0,0 +1,7 @@ +// The notes app's worker entry: one process per project. `startWorker` opens the project's store in its directory, +// connects to the server and loads the plugins' worker facets. + +import { startWorker } from "../../../../src/next-node/app/index.ts"; +import { API, KINDS } from "./start.ts"; + +startWorker({ api: API, kinds: KINDS, onError: (error) => console.error("worker:", error) }); diff --git a/packages/chord/test/next/examples/54-app-sessions/main.ts b/packages/chord/test/next/examples/54-app-sessions/main.ts new file mode 100644 index 000000000..775a54d84 --- /dev/null +++ b/packages/chord/test/next/examples/54-app-sessions/main.ts @@ -0,0 +1,62 @@ +// Sessions and device pairing (slice 13 §2), with example 53's notes app. A command-line client on another machine +// has no owner token file: it asks for a pairing code (`POST /_chord/pair`, public), the owner approves the code in a +// signed-in browser (`/_chord/pair/`, owner only), and the client's next poll returns its device token, once. +// The owner lists the sessions (`Sessions`, the kit's object; never a token or its hash) and revokes the device's. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/54-app-sessions/main.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { Sessions } from "../../../../src/next-node/app/index.ts"; +import { call } from "../53-app/client.ts"; +import { startNotes } from "../53-app/start.ts"; + +const base = await mkdtemp(join(tmpdir(), "notes-")); +const app = await startNotes(join(base, "data"), context); +const link = new URL(app.ownerLink()); +const port = Number(link.port); + +// 1. The owner signs in in the browser with the owner link (example 53). +const browser = { cookie: (await call(port, "POST", link.pathname, {}, { token: link.hash.slice(1) })).cookie ?? "" }; + +// 2. The client asks for a code and shows it; it keeps the poll secret to itself. +const { code, poll } = (await call(port, "POST", "/_chord/pair", {}, { label: "laptop CLI" })).body as { + code: string; + poll: string; +}; +console.log("1. pairing code:", code.replace(/[A-Z0-9]/g, "X")); +console.log(" poll:", JSON.stringify((await call(port, "POST", "/_chord/pair/poll", {}, { poll })).body)); + +// 3. Only the owner may see and approve it: an anonymous request is refused before the kit's route runs. +console.log("2. anonymous approval:", (await call(port, "POST", `/_chord/pair/${code}`)).status); +const confirm = await call(port, "GET", `/_chord/pair/${code}`, browser); +console.log( + " the owner's confirm page:", + confirm.status, + String(confirm.body).includes("laptop CLI") ? "(names the device)" : "", +); +console.log(" approved:", (await call(port, "POST", `/_chord/pair/${code}`, browser)).status); + +// 4. The client's next poll returns its token, once; it signs in with it as a bearer token. +const approved = (await call(port, "POST", "/_chord/pair/poll", {}, { poll })).body as { + status: string; + token: string; +}; +console.log("3. poll:", approved.status); +console.log(" poll again:", (await call(port, "POST", "/_chord/pair/poll", {}, { poll })).status); +const device = { authorization: `Bearer ${approved.token}` }; +console.log(" CLI GET /:", JSON.stringify((await call(port, "GET", "/", device)).body)); + +// 5. The owner lists the sessions (a server plugin would do this with `self.get(Sessions, {})`) and revokes the +// device's: its next request is refused. +const sessions = await app.host.get(Sessions, {}, context); +console.log("4. sessions:"); +for (const { kind, label } of sessions.list.value) console.log(` ${kind}: ${label}`); +const laptop = sessions.list.value.find(({ label }) => label === "laptop CLI"); +await sessions.revoke(laptop?.id ?? "", context); +console.log(" revoked; CLI GET /:", (await call(port, "GET", "/", device)).status); +sessions.release(); + +await app.close(context); +await rm(base, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/55-app-export/main.ts b/packages/chord/test/next/examples/55-app-export/main.ts new file mode 100644 index 000000000..25f9aff9b --- /dev/null +++ b/packages/chord/test/next/examples/55-app-export/main.ts @@ -0,0 +1,67 @@ +// Export and import a host's data (slice 13 §4, decision "App kit naming and export"), with example 53's notes app. +// `app.exportData` stops nothing: the project's running worker writes one archive of its own store (one consistent +// point) and files, with a manifest (app, API version, format); without a worker the server writes it. +// `app.importData` makes another address from it (refusing one that has data) and the next request starts its +// worker on the imported store and files. Archives of another app are refused. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/55-app-export/main.ts +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { method, obj, Project, remote, str } from "../53-app/app/v1.ts"; +import { startNotes } from "../53-app/start.ts"; + +// The notes plugin's contract, declared again with the API's builders (contracts are data: the same name, version and +// schema bind). The plugin's own files import `@notes/app/v1`, a host package this process has no module path for. +const Notes = remote("notes.notes", { + version: 1, + params: { id: str() }, + hostAddress: ({ id }) => Project({ id }), + variants: { + main: { + state: { text: str(), start: str() }, + methods: { + write: method(str()), + attach: method(obj({ name: str(), body: str() }), str()), + attachment: method(str(), str()), + }, + }, + }, +}); + +const base = await mkdtemp(join(tmpdir(), "notes-")); +const app = await startNotes(join(base, "data"), context); + +// 1. Project p1 gets a text and an attachment (its worker's store and files). The server reaches it as `Notes`. +const p1 = await app.host.get(Notes, { id: "p1" }, context); +await p1.write("the plan", context); +const attachment = await p1.attach({ name: "plan.txt", body: "step 1, step 2" }, context); +p1.release(); + +// 2. Export p1 while its worker runs: the worker writes the archive (mode 0600) and goes on. +const archive = join(base, "p1.tar"); +await app.exportData(Project({ id: "p1" }), archive, context); +console.log( + "1. exported:", + `${(await stat(archive)).size} bytes`, + `mode ${((await stat(archive)).mode & 0o777).toString(8)}`, +); + +// 3. Import it as p2: a copy with p1's text and attachment, its own worker and directory. +await app.importData(archive, Project({ id: "p2" }), context); +const p2 = await app.host.get(Notes, { id: "p2" }, context); +console.log("2. p2:", JSON.stringify(p2.text.value), JSON.stringify(await p2.attachment(attachment, context))); +await p2.write("p2's own plan", context); +p2.release(); +const again = await app.host.get(Notes, { id: "p1" }, context); +console.log(" p1 unchanged:", JSON.stringify(again.text.value)); +again.release(); + +// 4. An address that has data is never overwritten. +await app + .importData(archive, Project({ id: "p1" }), context) + .catch((error: Error) => console.log("3. refused:", error.message.replace(base, "…"))); + +await app.close(context); +await rm(base, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/56-web/app/principals.ts b/packages/chord/test/next/examples/56-web/app/principals.ts new file mode 100644 index 000000000..66d7d767d --- /dev/null +++ b/packages/chord/test/next/examples/56-web/app/principals.ts @@ -0,0 +1,7 @@ +// The board app's principals: its server, the kit's owner, and users. + +/** Who someone is in the board app. */ +export type BoardPrincipal = + | { readonly kind: "server" } + | { readonly kind: "owner" } + | { readonly kind: "user"; readonly id: string; readonly name: string }; diff --git a/packages/chord/test/next/examples/56-web/app/v1.ts b/packages/chord/test/next/examples/56-web/app/v1.ts new file mode 100644 index 000000000..a701cd305 --- /dev/null +++ b/packages/chord/test/next/examples/56-web/app/v1.ts @@ -0,0 +1,12 @@ +// The board app's API, version 1, shared by every host kind (`@board/app/v1`): the part of Chord its plugins use, the +// kit's `Themes` registry (examples 56 to 58 run this app). + +export { + constant, + defineFacet, + local, + method, + str, + type Theme, + Themes, +} from "../../../../../src/next/index.ts"; diff --git a/packages/chord/test/next/examples/56-web/app/v1/tui.ts b/packages/chord/test/next/examples/56-web/app/v1/tui.ts new file mode 100644 index 000000000..6d60493e0 --- /dev/null +++ b/packages/chord/test/next/examples/56-web/app/v1/tui.ts @@ -0,0 +1,6 @@ +// The board app's API v1 for terminal facets (`@board/app/v1/tui`): the `Lines` registry the terminal draws. + +import { defineRegistry } from "../../../../../../src/next/index.ts"; + +/** Lines of the terminal board, in plugin-name order. */ +export const Lines = defineRegistry("board.lines", { version: 1 }); diff --git a/packages/chord/test/next/examples/56-web/app/v1/web.ts b/packages/chord/test/next/examples/56-web/app/v1/web.ts new file mode 100644 index 000000000..fcf738694 --- /dev/null +++ b/packages/chord/test/next/examples/56-web/app/v1/web.ts @@ -0,0 +1,10 @@ +// The board app's API v1 for web facets (`@board/app/v1/web`): the `Panels` registry the page shows, one panel per +// contributing plugin. + +import { defineRegistry } from "../../../../../../src/next/index.ts"; + +/** A panel on the board page: its title and its text. */ +export type Panel = { readonly title: string; readonly text: string }; + +/** The page's panels, in plugin-name order. */ +export const Panels = defineRegistry("board.panels", { version: 1 }); diff --git a/packages/chord/test/next/examples/56-web/headless.ts b/packages/chord/test/next/examples/56-web/headless.ts new file mode 100644 index 000000000..590a5f116 --- /dev/null +++ b/packages/chord/test/next/examples/56-web/headless.ts @@ -0,0 +1,69 @@ +// A page without a browser, for examples 56 and 58: what `startPage` does, run in Node. It loads the page's combined +// file and runs its registered scripts (`vm` here; ``; + +export default defineFacet((host, self) => { + host.serve(Channels, () => ({ variant: "main", members: { names: constant(CHANNELS) } })); + self.contribute(Http, { + routes: { + "GET /": { public: true, handler: async (request) => ({ kind: "html", body: signIn(request.nonce) }) }, + "GET /files/:channel/:id": async (request, ctx) => { + const room = await ctx.get(Room, { id: request.params.channel! }); + const file = room.messages.value.find((message) => message.file?.id === request.params.id)?.file; + if (file === undefined) return { kind: "error", status: 404 }; + return { + kind: "bytes", + mime: file.mime, + size: file.size, + inline: file.mime.startsWith("image/") ? true : undefined, + download: file.mime.startsWith("image/") ? undefined : file.name, + read: (offset, length) => + room.read({ id: file.id, offset, length: Math.min(length, 1024 * 1024) }, BACKGROUND_CONTEXT), + }; + }, + }, + }); +}); diff --git a/packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts b/packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts new file mode 100644 index 000000000..7f78cf8cb --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts @@ -0,0 +1,78 @@ +// The chat plugin on a channel's worker: the room's messages in a document of the worker's store (kept by a local +// object: a persisted state needs a stable identity), and each principal's `Room` posting as them and sharing files +// through the kit's `Files` of this worker. + +import { + Attachment, + arr, + defineFacet, + defineState, + local, + Message, + method, + num, + obj, + opt, + Room, + str, +} from "@chat/app/v1"; +import { Files } from "@chat/app/v1/worker"; + +/** The newest messages a channel keeps. */ +const KEPT = 500; + +const Log = defineState("chat.log", { + schema: obj({ next: num(), messages: arr(Message) }), + version: 1, + initial: () => ({ next: 1, messages: [] }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 50, +}); + +const Store = local("chat.store", { + version: 1, + params: {}, + members: { messages: arr(Message), add: method(obj({ author: str(), text: str(), file: opt(Attachment) })) }, +}); + +export default defineFacet((host) => { + host.serve(Store, (_params, self) => { + const log = self.state(Log); + return { + messages: self.derive([log], ([value]) => value.messages), + add: async ({ author, text, file }, ctx) => { + await log.change((draft) => { + draft.messages.push({ + id: draft.next++, + author, + text, + at: Date.now(), + ...(file === undefined ? {} : { file }), + }); + if (draft.messages.length > KEPT) draft.messages.splice(0, draft.messages.length - KEPT); + }, ctx); + return null; + }, + }; + }); + host.serve(Room, (_params, principal, self) => { + const store = self.get(Store, {}); + const files = self.get(Files, {}); + const author = (principal as { readonly name?: string }).name ?? principal.kind; + return { + variant: "main", + members: { + messages: self.derive([store.messages], ([messages]) => messages), + post: (text, ctx) => store.add({ author, text }, ctx), + begin: (file, ctx) => files.begin(file, ctx), + append: (chunk, ctx) => files.append(chunk, ctx), + share: async ({ upload, text }, ctx) => { + const id = await files.finish(upload, ctx); + const { name, mime, size } = files.list.value.find((file) => file.id === id)!; + return store.add({ author, text, file: { id, name, mime, size } }, ctx); + }, + read: (range, ctx) => files.read(range, ctx), + }, + }; + }); +}); diff --git a/packages/chord/test/next/examples/59-chat/plugins/login.server.ts b/packages/chord/test/next/examples/59-chat/plugins/login.server.ts new file mode 100644 index 000000000..c6a5a9779 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/plugins/login.server.ts @@ -0,0 +1,5 @@ +// The kit's pick-a-name login, added to the chat as a one-line plugin. + +import { login } from "@chat/app/v1/server"; + +export default login; diff --git a/packages/chord/test/next/examples/59-chat/start.ts b/packages/chord/test/next/examples/59-chat/start.ts new file mode 100644 index 000000000..304449280 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/start.ts @@ -0,0 +1,76 @@ +// The chat's start: one server keeping the channel list and the surfaces, one worker per channel, a web page at `/chat` +// (the sign-in page at `/` comes from the chat plugin) and a terminal client. No harness: the app is its API, two +// plugins and its two client entries. + +import { cp, mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Context } from "../../../../src/context/index.ts"; +import { type App, startServer } from "../../../../src/next-node/app/index.ts"; +import { type PageAssets, pageScripts } from "../../../../src/next-node/surfaces/index.ts"; +import type { ChatPrincipal } from "./app/principals.ts"; +import * as v1server from "./app/v1/server.ts"; +import * as v1worker from "./app/v1/worker.ts"; +import * as v1 from "./app/v1.ts"; + +/** The chat's API, as the server and the channel workers have it. */ +export const API = { + name: "@chat/app", + versions: ["v1"], + modules: { "@chat/app/v1": v1, "@chat/app/v1/server": v1server, "@chat/app/v1/worker": v1worker }, +}; + +/** The chat's facet kinds. */ +export const KINDS = { + server: { platform: "node" }, + worker: { platform: "node" }, + web: { platform: "browser", packages: { react: {}, "react/jsx-runtime": {} } }, + tui: { platform: "node" }, +} as const; + +/** The chat page: the app's template around the surface's assets. */ +function page(request: { readonly nonce: string }, assets: PageAssets): string { + return `Chat${pageScripts(assets, request.nonce)}
`; +} + +/** A data directory and a copy of the chat's plugins, in a new temporary directory. */ +export async function chatDirectories(): Promise<{ + readonly base: string; + readonly dataDir: string; + readonly root: string; +}> { + const base = await mkdtemp(join(tmpdir(), "chat-")); + const root = join(base, "plugins"); + await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + return { base, dataDir: join(base, "data"), root }; +} + +/** Starts the chat on a free loopback port. */ +export function startChat(dataDir: string, root: string, context: Context): Promise> { + return startServer( + { + name: "chat", + dataDir, + root, + listen: { host: "127.0.0.1", port: 0 }, + api: API, + kinds: KINDS, + principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", channel: address.key }) }, + workers: [ + { + kind: v1.Channel, + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + start: () => ({}), + // Anyone signed in may join a channel. + authorize: (principal) => principal.kind === "user" || principal.kind === "owner", + }, + ], + web: { entry: fileURLToPath(new URL("./web/main.ts", import.meta.url)), page, paths: ["/chat"] }, + tui: { entry: fileURLToPath(new URL("./tui/main.ts", import.meta.url)) }, + onError: (error) => console.error("chat:", error.kind, error.error), + }, + context, + ); +} diff --git a/packages/chord/test/next/examples/59-chat/tui/main.ts b/packages/chord/test/next/examples/59-chat/tui/main.ts new file mode 100644 index 000000000..b3dd20f52 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/tui/main.ts @@ -0,0 +1,90 @@ +// The chat in a terminal (the terminal base bundle, run in the launcher's sandbox): the channels (Tab switches), the +// current room's messages (an image as a placeholder, other files as their link), and a line to type into. Enter +// posts; `/attach ` asks the launcher for a file of this machine (it asks the user) and shares it in chunks; +// Ctrl-C quits. + +import type { Held } from "../../../../../src/next/index.ts"; +import { type AttachedFile, startTerminalRuntime, TuiScope } from "../../../../../src/next-tui/index.ts"; +import * as v1 from "../app/v1.ts"; + +const { BACKGROUND_CONTEXT: context, Channels, Room } = v1; + +/** The size of one upload chunk. */ +const CHUNK = 256 * 1024; + +/** Shares `file` in `room`, in chunks. */ +async function share(room: Held, file: AttachedFile): Promise { + const mime = /\.(png|jpe?g|gif|webp)$/i.test(file.name) + ? `image/${file.name.split(".").pop()!.toLowerCase()}` + : "application/octet-stream"; + const id = await room.begin({ name: file.name, mime }, context); + for (let offset = 0; offset < file.bytes.length; offset += CHUNK) { + await room.append({ upload: id, offset, bytes: file.bytes.subarray(offset, offset + CHUNK) }, context); + } + await room.share({ upload: id, text: "" }, context); +} + +/** One message as a line. */ +function line(message: v1.Message, channel: string, theme: { fg(role: string, text: string): string }): string { + const { file } = message; + const attached = + file === undefined + ? "" + : file.mime.startsWith("image/") + ? ` ${theme.fg("muted", `[image: ${file.name}]`)}` + : ` ${theme.fg("accent", `[${file.name}] /files/${channel}/${file.id}`)}`; + return `${theme.fg("accent", message.author)} ${message.text}${attached}`; +} + +void startTerminalRuntime({ + modules: { "@chat/app/v1": v1 }, + render: (runtime) => { + const { terminal, theme } = runtime; + const scope = new TuiScope(runtime.host, () => draw()); + const channels = scope.watch(scope.hold(Channels, {}).names); + // Each channel's room held once, on first view. + const rooms = new Map; messages: () => readonly v1.Message[] | undefined }>(); + const roomOf = (id: string) => { + let held = rooms.get(id); + if (held === undefined) { + const room = scope.hold(Room, { id }); + held = { room, messages: scope.watch(room.messages) }; + rooms.set(id, held); + } + return held; + }; + let channel = "general"; + let draft = ""; + const draw = (): void => { + const tabs = (channels() ?? []).map((name) => + name === channel ? theme.fg("accent", `#${name}`) : `#${name}`, + ); + const shown = (roomOf(channel).messages() ?? []) + .slice(-(terminal.rows - 3)) + .map((message) => line(message, channel, theme)); + terminal.write(`\x1b[2J\x1b[H${tabs.join(" ")}\r\n${shown.join("\r\n")}\r\n> ${draft}`); + }; + const send = (text: string): void => { + if (text.startsWith("/attach ")) { + const { room } = roomOf(channel); + void runtime + .attach(text.slice(8).trim()) + .then((file) => (file === undefined ? undefined : share(room, file))); + } else if (text !== "") void roomOf(channel).room.post(text, context); + }; + terminal.onInput((data) => { + if (data === "\x03") runtime.exit(0); + else if (data === "\t") { + const names = channels() ?? []; + channel = names[(names.indexOf(channel) + 1) % names.length] ?? channel; + } else if (data === "\r") { + send(draft); + draft = ""; + } else if (data === "\x7f") draft = draft.slice(0, -1); + else if (!data.startsWith("\x1b")) draft += data; + draw(); + }); + theme.onChange(draw); + draw(); + }, +}); diff --git a/packages/chord/test/next/examples/59-chat/web/main.ts b/packages/chord/test/next/examples/59-chat/web/main.ts new file mode 100644 index 000000000..478f2b218 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/web/main.ts @@ -0,0 +1,95 @@ +// The chat's page (the web base bundle): the channels as tabs, the current room's messages (images inline, other files +// as links), a message box and a file picker. A file goes up in chunks through the room (into its worker's files). + +import * as React from "react"; +import { createElement as h, useState } from "react"; +import * as jsx from "react/jsx-runtime"; +import { createRoot } from "react-dom/client"; +import type { Held } from "../../../../../src/next/index.ts"; +import { HostProvider, useHold, useWatch } from "../../../../../src/next-react/index.ts"; +import { startPage } from "../../../../../src/next-web/index.ts"; +import * as v1 from "../app/v1.ts"; +import "./style.css"; + +const { BACKGROUND_CONTEXT: context, Channels, Room } = v1; + +/** The size of one upload chunk. */ +const CHUNK = 256 * 1024; + +/** Sends `file` to `room` in chunks, then shares it. */ +async function upload(room: Held, file: File): Promise { + const id = await room.begin({ name: file.name, mime: file.type || "application/octet-stream" }, context); + for (let offset = 0; offset < file.size; offset += CHUNK) { + const bytes = new Uint8Array(await file.slice(offset, offset + CHUNK).arrayBuffer()); + await room.append({ upload: id, offset, bytes }, context); + } + await room.share({ upload: id, text: "" }, context); +} + +function Message({ channel, message }: { channel: string; message: v1.Message }) { + const { file } = message; + const url = file === undefined ? "" : `/files/${channel}/${file.id}`; + const body = + file === undefined + ? null + : file.mime.startsWith("image/") + ? h("img", { src: url, alt: file.name }) + : h("a", { href: url }, `${file.name} (${file.size} bytes)`); + return h("li", null, h("b", null, message.author), " ", message.text, body); +} + +function RoomView({ channel }: { channel: string }) { + const room = useHold(Room, { id: channel }); + const messages = useWatch(room?.messages) ?? []; + const [text, setText] = useState(""); + const send = (event: { preventDefault(): void }) => { + event.preventDefault(); + if (room === undefined || text === "") return; + void room.post(text, context); + setText(""); + }; + return h( + "section", + null, + h("ul", null, ...messages.map((message) => h(Message, { key: message.id, channel, message }))), + h( + "form", + { onSubmit: send }, + h("input", { value: text, onChange: (event: { target: { value: string } }) => setText(event.target.value) }), + h("input", { + type: "file", + onChange: (event: { target: { files: FileList | null } }) => { + const file = event.target.files?.[0]; + if (room !== undefined && file !== undefined) void upload(room, file); + }, + }), + ), + ); +} + +function Chat() { + const channels = useWatch(useHold(Channels, {})?.names) ?? []; + const [channel, setChannel] = useState("general"); + return h( + "main", + null, + h( + "nav", + null, + ...channels.map((name) => + h( + "button", + { key: name, type: "button", onClick: () => setChannel(name), "aria-pressed": name === channel }, + `#${name}`, + ), + ), + ), + h(RoomView, { key: channel, channel }), + ); +} + +void startPage({ + connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), + modules: { "@chat/app/v1": v1, react: React, "react/jsx-runtime": jsx }, + render: (host) => createRoot(document.getElementById("root")!).render(h(HostProvider, { host }, h(Chat))), +}); diff --git a/packages/chord/test/next/examples/59-chat/web/style.css b/packages/chord/test/next/examples/59-chat/web/style.css new file mode 100644 index 000000000..ccfea4873 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/web/style.css @@ -0,0 +1,7 @@ +/* The chat's look: the active theme's colours (the kit writes them on :root). */ +body { margin: 0; font: 15px system-ui, sans-serif; background: var(--bg); color: var(--fg); } +main { max-width: 40rem; margin: 1rem auto; } +nav button[aria-pressed="true"] { color: var(--accent); } +ul { list-style: none; padding: 0; } +li img { display: block; max-width: 16rem; margin-top: 0.25rem; } +a { color: var(--accent); } diff --git a/packages/chord/test/next/examples/59-chat/worker.ts b/packages/chord/test/next/examples/59-chat/worker.ts new file mode 100644 index 000000000..982eefcd2 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/worker.ts @@ -0,0 +1,6 @@ +// A channel's worker: the kit's `startWorker` with the chat's API. + +import { startWorker } from "../../../../src/next-node/app/index.ts"; +import { API, KINDS } from "./start.ts"; + +startWorker({ api: API, kinds: KINDS, onError: (error) => console.error("channel:", error) }); diff --git a/packages/chord/test/next/examples/60-session-sync/main.ts b/packages/chord/test/next/examples/60-session-sync/main.ts new file mode 100644 index 000000000..04c922d8f --- /dev/null +++ b/packages/chord/test/next/examples/60-session-sync/main.ts @@ -0,0 +1,180 @@ +// Session sync as decided (decisions S3, S5, S7, W9, "The outbox is optional, internal and versioned", "p3 meeting, +// 2026-11-02"), played with Chord's real pieces: a laptop store with an outbox, `store.outbox.read`/`trim`, +// `store.export` with `encodeJsonl`, `importStore`, and batch apply, `applyBatch` (internal: exported from +// `chord/next` marked internal, imported here from `src/next/store/apply-batch.ts`; it takes the `ChordStore` behind +// `openStore`'s `Store`). Radius and the laptop's sync client are stand-ins written for this example (`radius.ts`, +// `sync.ts`), not Chord APIs: the sync layer is not built yet (slices.md, "The sync layer"). +// Steps: 1. a session on the laptop; 2. the handshake and a first upload; 3. a second sync moves only the tail; +// 4. a bootstrap from an archive for a session whose outbox was turned on late; 5. "continue on Radius" and the epoch +// fence (the split), then the same without the fence (silent divergence); 6. back to the laptop with a pull. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/60-session-sync/main.ts +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { decodeJsonl, importStore, openStore, type Store, sqliteBackend } from "../../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../../src/sqlite/node.ts"; +import { RadiusStandIn } from "./radius.ts"; +import { describe, withSession } from "./session.ts"; +import { archiveOf, type LocalSession, pull, push, type Synced, sync } from "./sync.ts"; + +const radius = new RadiusStandIn(); +const directory = await mkdtemp(join(tmpdir(), "chord-sync-")); + +/** One sync's result as a line. */ +function shown(result: Synced): string { + switch (result.kind) { + case "uploaded": + return `uploaded ${result.from}..${result.to}, Radius at ${result.lastSeq}, trimmed through it`; + case "nothing": + return `nothing to send, Radius at ${result.lastSeq}`; + case "bootstrapped": + return `archive uploaded (${result.bytes} bytes), Radius imported it at ${result.lastSeq}`; + case "behind": + return `Radius is ahead at ${result.lastSeq}`; + case "split": + return `refused for an old epoch: this copy is now ${result.session.id}`; + case "refused": + return `refused: ${result.message}`; + } +} + +/** A new in-memory store (outbox on) imported from archive bytes. */ +async function copyOf(bytes: Uint8Array): Promise { + const backend = sqliteBackend(await openNodeSqliteDatabase(":memory:")); + return importStore(backend, decodeJsonl(Readable.from([bytes])), context, { outbox: true }); +} + +/** Radius's view of a session as a line. */ +function atRadius(id: string): string { + const known = radius.handshake()[id]!; + return `Radius ${id}: lastSeq ${known.lastSeq}, log ${radius.logRange(id)}, epoch ${known.epoch} (${known.driver})`; +} + +try { + // 1. The laptop: SQLite in memory with the outbox on from its first commit; one session, a few commits. + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, { outbox: true }); + const s1: LocalSession = { id: "s1", store, epoch: 1, acked: 0 }; + await withSession(store, async (session) => { + await session.rename("Fix the parser", context); + await session.say({ role: "user", text: "the parser drops comments" }, context); + await session.say({ role: "assistant", text: "found it in lexer.ts" }, context); + }); + console.log("1. laptop s1: lastSeq", store.lastSeq, "outbox floor", store.outbox!.floor); + + // 2. The handshake: Radius has no s1 (lastSeq 0), so the whole outbox goes, 1..lastSeq; then trimmed. + console.log("2. sync:", shown(await sync(s1, radius))); + console.log(" ", atRadius("s1"), "- laptop outbox floor", store.outbox!.floor); + + // 3. Two more commits; the next sync sends only them. + await withSession(store, async (session) => { + await session.say({ role: "user", text: "and strings too" }, context); + await session.say({ role: "assistant", text: "fixed both" }, context); + }); + console.log("3. laptop s1: lastSeq", store.lastSeq, "- sync:", shown(await sync(s1, radius))); + console.log(" ", atRadius("s1")); + console.log(" laptop:", await describe(store)); + console.log(" Radius:", await describe(radius.replica("s1"))); + + // 4. Session s2 ran without an outbox (a file here, since it is reopened), then sync was turned on: the outbox starts + // after its lastSeq, so Radius, which has nothing, needs an archive first; batches go on from there. + const path = join(directory, "s2.sqlite"); + const before = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + await withSession(before, async (session) => { + await session.rename("Release notes", context); + await session.say({ role: "user", text: "draft the notes for 1.4" }, context); + }); + await before.close(context); + const store2 = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: true }); + const s2: LocalSession = { id: "s2", store: store2, epoch: 1, acked: 0 }; + console.log( + "4. laptop s2: lastSeq", + store2.lastSeq, + "outbox floor", + store2.outbox!.floor, + "- sync:", + shown(await sync(s2, radius)), + ); + await withSession(store2, async (session) => { + await session.say({ role: "assistant", text: "here is a draft" }, context); + }); + console.log(" laptop s2: lastSeq", store2.lastSeq, "- sync:", shown(await sync(s2, radius))); + console.log(" ", atRadius("s2")); + console.log(" Radius:", await describe(radius.replica("s2"))); + await store2.close(context); + + // 5. "Continue on Radius": the laptop uploads its tail (nothing left here), Radius bumps the epoch and drives, + // writing a commit on its replica. The laptop keeps its copy as a replica and writes nothing. + const handoff = await archiveOf(store); + console.log( + "5. laptop s1 sync:", + shown(await sync(s1, radius)), + "- Radius takes over, epoch", + radius.takeOver("s1"), + ); + await radius.drive("s1", (replica) => + withSession(replica, async (session) => { + await session.say({ role: "assistant", text: "(on Radius) added a test" }, context); + }), + ); + console.log(" ", atRadius("s1")); + // A laptop process that missed the handoff (here a copy of the laptop's store at it, so the laptop itself can take + // the session back in step 6) writes at epoch 1 and uploads the new commit after its last acknowledgement. + const stale: LocalSession = { id: "s1", store: await copyOf(handoff), epoch: 1, acked: 6 }; + await withSession(stale.store, async (session) => { + await session.say({ role: "assistant", text: "(stale laptop) added a different test" }, context); + }); + const fenced = await push(stale, radius); + console.log(" stale laptop: lastSeq", stale.store.lastSeq, "- push:", shown(fenced)); + // The split (S7): no rollback; the copy is a new session with its origin, synced on its own from an archive. + if (fenced.kind === "split") { + const split = fenced.session; + console.log( + " split", + split.id, + "origin", + JSON.stringify(split.origin), + "- sync:", + shown(await sync(split, radius)), + ); + console.log(" ", atRadius(split.id)); + } + await stale.store.close(context); + + // 5b. The same without the epoch fence: the stale upload of seq 7 does not follow Radius's 7 and is refused, so far + // so good; but the next handshake finds both at 7, sends nothing and trims the laptop's commit 7 as acknowledged. + radius.checkEpochs = false; + const unfenced: LocalSession = { id: "s1", store: await copyOf(handoff), epoch: 1, acked: 6 }; + await withSession(unfenced.store, async (session) => { + await session.say({ role: "assistant", text: "(stale laptop) added a different test" }, context); + }); + console.log("5b. no epoch check - stale push:", shown(await push(unfenced, radius))); + console.log(" stale sync:", shown(await sync(unfenced, radius)), "- outbox floor", unfenced.store.outbox!.floor); + console.log(" stale: ", await describe(unfenced.store)); + console.log(" Radius:", await describe(radius.replica("s1"))); + await unfenced.store.close(context); + radius.checkEpochs = true; + + // 6. Back to the laptop: it takes the epoch (Radius stops driving), pulls Radius's commits after its lastSeq + // (batch apply on the laptop's store; no host open on it), and drives again. + s1.epoch = radius.handBack("s1", "laptop"); + const pulled = await pull(s1, radius); + console.log(`6. laptop takes epoch ${s1.epoch}, pulls ${pulled.from}..${pulled.to}: lastSeq ${store.lastSeq}`); + const [laptopView, radiusView] = [await describe(store), await describe(radius.replica("s1"))]; + console.log(" laptop:", laptopView); + console.log(" Radius:", radiusView); + console.log(" same values:", laptopView === radiusView); + await withSession(store, async (session) => { + await session.rename("Fix the parser (done)", context); + }); + console.log(" laptop writes, sync:", shown(await sync(s1, radius))); + console.log(" ", atRadius("s1")); + console.log(" same values:", (await describe(store)) === (await describe(radius.replica("s1")))); + + await store.close(context); +} finally { + await radius.close(); + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/examples/60-session-sync/radius.ts b/packages/chord/test/next/examples/60-session-sync/radius.ts new file mode 100644 index 000000000..177ca5cf9 --- /dev/null +++ b/packages/chord/test/next/examples/60-session-sync/radius.ts @@ -0,0 +1,159 @@ +// STAND-IN for Radius, not a Chord API: what example 60 needs of the server side of session sync, in memory. Per +// session id it keeps a replica store (SQLite in memory, as a Durable Object's SQLite would be), the commit log (what +// R2 would hold; the replica's outbox is moved into it and trimmed after every change), the driver and its epoch. +// Batches and archives cross a "wire" as JSON and JSONL bytes, so the replica checks them as a trust boundary. +// `checkEpochs = false` turns the epoch fence off, to show what the gap analysis warned about. +import { Readable } from "node:stream"; +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { ChordError } from "../../../../src/next/data/index.ts"; +// Internal: the sync layer's batch apply (exported from `chord/next` marked internal) takes the `ChordStore` that +// `openStore` returns behind the app-facing `Store` type. +import { applyBatch } from "../../../../src/next/store/apply-batch.ts"; +import { + decodeJsonl, + importStore, + type OutboxCommit, + openStore, + type Store, + sqliteBackend, +} from "../../../../src/next/store/index.ts"; +import type { ChordStore } from "../../../../src/next/store/store.ts"; +import { openNodeSqliteDatabase } from "../../../../src/sqlite/node.ts"; + +/** What the handshake reports per session: Radius's applied head, the epoch and who drives. */ +export type Known = { readonly lastSeq: number; readonly epoch: number; readonly driver: string }; + +/** A batch as it travels: the commits `from..to` and the uploader's epoch (S5: every batch carries `from` and `to`). */ +export type Batch = { + readonly from: number; + readonly to: number; + readonly epoch: number; + readonly commits: readonly OutboxCommit[]; +}; + +/** An upload's answer: applied (the new head is the acknowledgement) or refused, for an old epoch or by the replica. */ +export type Uploaded = + | { readonly kind: "applied"; readonly lastSeq: number; readonly epoch: number } + | { readonly kind: "refused"; readonly reason: "epoch"; readonly epoch: number } + | { readonly kind: "refused"; readonly reason: "batch"; readonly message: string }; + +type Entry = { + readonly replica: Store; + /** The commit log (R2's part): every commit after `logFloor` (the seq it was bootstrapped at). */ + readonly log: OutboxCommit[]; + readonly logFloor: number; + epoch: number; + driver: string; +}; + +/** The stand-in: sessions by id, for one owner. */ +export class RadiusStandIn { + /** The epoch fence (S7); off only to show the divergence without it. */ + checkEpochs = true; + readonly #sessions = new Map(); + + /** The handshake: every session Radius has, with its `lastSeq`, epoch and driver. */ + handshake(): Record { + const known: Record = {}; + for (const [id, { replica, epoch, driver }] of this.#sessions) + known[id] = { lastSeq: replica.lastSeq, epoch, driver }; + return known; + } + + /** Applies an uploaded batch (JSON) to the session's replica; a first batch from seq 1 makes the session. */ + async upload(id: string, driver: string, wire: string): Promise { + const batch = JSON.parse(wire) as Batch; + let entry = this.#sessions.get(id); + if (entry === undefined && batch.from === 1) entry = this.#create(id, driver, await openReplica()); + if (entry === undefined) return { kind: "refused", reason: "batch", message: "unknown session: bootstrap first" }; + if (this.checkEpochs && batch.epoch !== entry.epoch) + return { kind: "refused", reason: "epoch", epoch: entry.epoch }; + try { + await applyBatch(entry.replica as ChordStore, batch.commits, context); + } catch (error) { + if (!(error instanceof ChordError)) throw error; + return { kind: "refused", reason: "batch", message: error.message }; + } + await this.#moveToLog(entry); + return { kind: "applied", lastSeq: entry.replica.lastSeq, epoch: entry.epoch }; + } + + /** Makes the session from an archive (JSONL bytes): its replica imported, the log starting at its `lastSeq`. */ + async bootstrap(id: string, driver: string, bytes: Uint8Array): Promise { + const backend = sqliteBackend(await openNodeSqliteDatabase(":memory:")); + const replica = await importStore(backend, decodeJsonl(Readable.from([bytes])), context, { outbox: true }); + const entry = this.#create(id, driver, replica); + return { lastSeq: replica.lastSeq, epoch: entry.epoch, driver }; + } + + /** "Continue on Radius": a new epoch, Radius drives; an upload at the old epoch is refused from now on. */ + takeOver(id: string): number { + const entry = this.#entry(id); + entry.epoch += 1; + entry.driver = "radius"; + return entry.epoch; + } + + /** Radius as the driver: `work` writes on the replica (a host on it, closed again), its commits go to the log. */ + async drive(id: string, work: (replica: Store) => Promise): Promise { + const entry = this.#entry(id); + if (entry.driver !== "radius") throw new Error(`radius: ${id} is driven by ${entry.driver}`); + await work(entry.replica); + await this.#moveToLog(entry); + } + + /** "Back to the laptop": a new epoch for `driver`; Radius stops writing. */ + handBack(id: string, driver: string): number { + const entry = this.#entry(id); + entry.epoch += 1; + entry.driver = driver; + return entry.epoch; + } + + /** The log's commits after `afterSeq` (JSON), for a pull. */ + pull(id: string, afterSeq: number): string { + const entry = this.#entry(id); + if (afterSeq < entry.logFloor) throw new Error(`radius: ${id}'s log starts after ${entry.logFloor}`); + return JSON.stringify(entry.log.filter((commit) => commit.seq > afterSeq)); + } + + /** The session's replica, to read it in the example. */ + replica(id: string): Store { + return this.#entry(id).replica; + } + + /** The seqs the log holds, as `from..to`. */ + logRange(id: string): string { + const { log, logFloor } = this.#entry(id); + return log.length === 0 ? `empty after ${logFloor}` : `${log[0]!.seq}..${log.at(-1)!.seq}`; + } + + /** Closes every replica. */ + async close(): Promise { + for (const { replica } of this.#sessions.values()) await replica.close(context); + } + + #create(id: string, driver: string, replica: Store): Entry { + const entry: Entry = { replica, log: [], logFloor: replica.lastSeq, epoch: 1, driver }; + this.#sessions.set(id, entry); + return entry; + } + + #entry(id: string): Entry { + const entry = this.#sessions.get(id); + if (entry === undefined) throw new Error(`radius: no session ${id}`); + return entry; + } + + /** The replica's new commits into the log (the DO writing its commit log to R2), then trimmed from its outbox. */ + async #moveToLog(entry: Entry): Promise { + const outbox = entry.replica.outbox!; + entry.log.push(...(await outbox.read(outbox.floor, 10_000, context))); + await outbox.trim(entry.replica.lastSeq, context); + } +} + +/** A replica as Radius makes it for a session uploaded from seq 1: empty, its outbox on. */ +async function openReplica(): Promise { + return openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, { outbox: true }); +} diff --git a/packages/chord/test/next/examples/60-session-sync/session.ts b/packages/chord/test/next/examples/60-session-sync/session.ts new file mode 100644 index 000000000..b1d4c9e30 --- /dev/null +++ b/packages/chord/test/next/examples/60-session-sync/session.ts @@ -0,0 +1,83 @@ +// The session example 60 syncs: one object per host with a persisted title document and a `messages` table, and a +// helper running a host on a store for the length of some work (one open host per store; a closed host holds no +// claims, so a batch may apply to its store afterwards). +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { arr, int, local, method, obj, str } from "../../../../src/next/contracts/index.ts"; +import { defineState } from "../../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type Reference } from "../../../../src/next/host/index.ts"; +import { defineTable, query, type Store } from "../../../../src/next/store/index.ts"; + +const Messages = defineTable("messages", { + version: 1, + columns: { n: int(), role: str(), text: str() }, + key: ["n"], +}); + +const Title = defineState("title", { + schema: obj({ text: str() }), + version: 1, + initial: () => ({ text: "untitled" }), + persist: true, + checkpointWhen: () => false, +}); + +/** The session's contract: its title, a message (one commit each), a rename (one commit) and the transcript. */ +export const Session = local("example.session", { + version: 1, + params: {}, + members: { + title: obj({ text: str() }), + say: method(obj({ role: str(), text: str() }), int()), + rename: method(str()), + transcript: method(obj({}), arr(str())), + }, +}); + +const facet = defineFacet((host) => { + host.serve(Session, (_, self) => { + const messages = self.table(Messages); + const title = self.state(Title); + const all = query(Messages).orderBy((m) => m.n, "asc"); + return { + title, + say: async ({ role, text }, ctx) => { + const n = (await messages.count(all, ctx)) + 1; + await self.tx(async (tx) => tx.put(messages, { n, role, text }), ctx); + return n; + }, + rename: async (text, ctx) => { + await title.change((draft) => { + draft.text = text; + }, ctx); + return null; + }, + transcript: async (_, ctx) => { + const lines: string[] = []; + for await (const m of messages.rows(all, ctx)) lines.push(`${m.n} ${m.role}: ${m.text}`); + return lines; + }, + }; + }); +}); + +/** Runs a host serving the session on `store` until `work` is done, then closes the host (the store stays open). */ +export async function withSession( + store: Store, + work: (session: Reference) => Promise, +): Promise { + const host = createHost({ store, onError: (error) => console.error("background error:", error) }); + try { + await host.load([{ name: "@example/session", facet }], context); + return await work(await host.get(Session, {}, context)); + } finally { + await host.close(context); + } +} + +/** The session as one line: its title and its messages. */ +export function describe(store: Store): Promise { + return withSession(store, async (session) => { + const lines = await session.transcript({}, context); + return `"${session.title.value.text}" [${lines.join(" | ")}]`; + }); +} diff --git a/packages/chord/test/next/examples/60-session-sync/sync.ts b/packages/chord/test/next/examples/60-session-sync/sync.ts new file mode 100644 index 000000000..e88c7e0b9 --- /dev/null +++ b/packages/chord/test/next/examples/60-session-sync/sync.ts @@ -0,0 +1,101 @@ +// STAND-IN for the laptop's sync layer, not a Chord API: the decided handshake (ask Radius for `lastSeq`, send the +// outbox from `lastSeq + 1`, trim through the answer), the bootstrap from an archive, the eager upload of new commits +// after the last acknowledgement (S5), the split on an old epoch (S7) and the pull of Radius's commits. Built on +// Chord's real pieces: `store.outbox.read`/`trim`, `store.export`, `encodeJsonl`, and `applyBatch` (internal). +import { BACKGROUND_CONTEXT as context } from "../../../../src/context/index.ts"; +import { applyBatch } from "../../../../src/next/store/apply-batch.ts"; +import { encodeJsonl, type OutboxCommit, type Store } from "../../../../src/next/store/index.ts"; +import type { ChordStore } from "../../../../src/next/store/store.ts"; +import type { Batch, RadiusStandIn } from "./radius.ts"; + +/** A session on the laptop: its id, store, the epoch it drives in (a new session: 1) and the `lastSeq` Radius last acknowledged. */ +export type LocalSession = { + readonly id: string; + readonly store: Store; + epoch: number; + acked: number; + /** A split-off copy's origin: the session it was and the last seq they share (S7). */ + readonly origin?: { readonly id: string; readonly seq: number }; +}; + +/** What a sync or a push did. */ +export type Synced = + | { readonly kind: "uploaded"; readonly from: number; readonly to: number; readonly lastSeq: number } + | { readonly kind: "nothing"; readonly lastSeq: number } + | { readonly kind: "bootstrapped"; readonly lastSeq: number; readonly bytes: number } + | { readonly kind: "behind"; readonly lastSeq: number } + | { readonly kind: "split"; readonly session: LocalSession } + | { readonly kind: "refused"; readonly message: string }; + +const DRIVER = "laptop"; + +/** The handshake, then the outbox from Radius's `lastSeq + 1`; an archive when Radius has less than the outbox. */ +export async function sync(session: LocalSession, radius: RadiusStandIn): Promise { + const { store } = session; + const outbox = store.outbox!; + const known = radius.handshake()[session.id]; + if (known === undefined && outbox.floor > 0) return bootstrap(session, radius); + if (known !== undefined && known.lastSeq < outbox.floor) return bootstrap(session, radius); + // Radius ahead: a pull-back (S3, same epoch) or a split (S7); step 6 pulls explicitly. + if (known !== undefined && known.lastSeq > store.lastSeq) return { kind: "behind", lastSeq: known.lastSeq }; + session.acked = known?.lastSeq ?? 0; + await outbox.trim(session.acked, context); + return push(session, radius); +} + +/** Uploads the outbox after the last acknowledgement, without asking first (an eager or resubmitted upload, S5). */ +export async function push(session: LocalSession, radius: RadiusStandIn): Promise { + const { store } = session; + const commits = await store.outbox!.read(session.acked, 10_000, context); + if (commits.length === 0) return { kind: "nothing", lastSeq: session.acked }; + const batch: Batch = { from: commits[0]!.seq, to: commits.at(-1)!.seq, epoch: session.epoch, commits }; + const answer = await radius.upload(session.id, DRIVER, JSON.stringify(batch)); + switch (answer.kind) { + case "applied": + session.acked = answer.lastSeq; + session.epoch = answer.epoch; + await store.outbox!.trim(answer.lastSeq, context); + return { kind: "uploaded", from: batch.from, to: batch.to, lastSeq: answer.lastSeq }; + case "refused": + if (answer.reason === "batch") return { kind: "refused", message: answer.message }; + // S7: no rollback; this copy goes on as a new session that shares the old one's commits through `acked`. + return { + kind: "split", + session: { + id: `${session.id}~split@${session.acked}`, + store, + epoch: 1, + acked: 0, + origin: { id: session.id, seq: session.acked }, + }, + }; + } +} + +/** Uploads the store's archive; Radius imports it and batches go on from its `lastSeq`. */ +async function bootstrap(session: LocalSession, radius: RadiusStandIn): Promise { + const bytes = await archiveOf(session.store); + const known = await radius.bootstrap(session.id, DRIVER, bytes); + session.acked = known.lastSeq; + session.epoch = known.epoch; + await session.store.outbox!.trim(known.lastSeq, context); + return { kind: "bootstrapped", lastSeq: known.lastSeq, bytes: bytes.length }; +} + +/** Applies Radius's commits after the local `lastSeq` (no host may be open on the store: its claims refuse a batch). */ +export async function pull(session: LocalSession, radius: RadiusStandIn): Promise<{ from: number; to: number }> { + const commits = JSON.parse(radius.pull(session.id, session.store.lastSeq)) as OutboxCommit[]; + const from = session.store.lastSeq + 1; + await applyBatch(session.store as ChordStore, commits, context); + // Batch apply writes the commits to this store's outbox too; Radius has them, so they are acknowledged. + session.acked = session.store.lastSeq; + await session.store.outbox!.trim(session.acked, context); + return { from, to: session.store.lastSeq }; +} + +/** The store's archive as JSONL bytes (the commit queue held while it is read). */ +export async function archiveOf(store: Store): Promise { + const parts: Uint8Array[] = []; + for await (const part of encodeJsonl(store.export(context))) parts.push(part); + return Buffer.concat(parts); +} diff --git a/packages/chord/test/next/host/activity.test.ts b/packages/chord/test/next/host/activity.test.ts index 405089d6d..63ac47af6 100644 --- a/packages/chord/test/next/host/activity.test.ts +++ b/packages/chord/test/next/host/activity.test.ts @@ -6,9 +6,14 @@ import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { bool, int, local, method, remote } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { constant } from "../../../src/next/state/index.ts"; -import { fakeTimers, flushed, framesOf, linked, loaded, Notes, serveNotes, transported } from "../support.ts"; +import { fakeTimers, flushed, framesOf, linked, loaded, Notes, Server, serveNotes, transported } from "../support.ts"; -const Kept = remote("app.kept", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); +const Kept = remote("app.kept", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); describe("host.activity", () => { it("counts holds arriving over links and served here, not this host's own references", async () => { @@ -22,6 +27,19 @@ describe("host.activity", () => { expect(server.activity.value.references).toBe(0); }); + it("counts a hold placed here from its arrival, bound or not: a missing provider keeps the worker alive (M§10)", async () => { + const { server, client } = await linked(() => {}); + const missing = client.hold(Notes, { session: "S1" }, context); + await transported(); + expect(missing.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(server.activity.value.references).toBe(1); + // Presence lists bound references only. + expect(server.references(Notes).value).toEqual([]); + missing.release(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + it("counts method calls running on this host, local and remote, until they settle", async () => { const { server, client } = await linked((h) => void serveNotes(h)); const remoteRef = await client.get(Notes, { session: "S1" }, context); diff --git a/packages/chord/test/next/host/adapted.test.ts b/packages/chord/test/next/host/adapted.test.ts new file mode 100644 index 000000000..c95e83b11 --- /dev/null +++ b/packages/chord/test/next/host/adapted.test.ts @@ -0,0 +1,1499 @@ +// Tests of adapted members (Q10, §22, slice 7b §7): consumers bound at an older version a provider accepts read its +// state mapped (diffs per commit), call it with mapped arguments and results or its replacement of a removed method, +// read keyed members mapped per key, and are refused a member the provider refuses. + +import { describe, expect, it } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, evolve, int, keyed, local, method, obj, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { AdaptedMembers } from "../../../src/next/host/adapted.ts"; +import { createHost, defineFacet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + changesOf, + delivered, + expectCollected, + flushed, + instancesOf, + metered, + rejection, + valuesOf, +} from "../support.ts"; + +/** v1: a list of texts, `add` taking a text and returning the count, `clear`, keyed notes of a title. */ +const NotesV1 = local("app.notes", { + version: 1, + params: {}, + members: { + items: arr(str()), + add: method(obj({ text: str() }), int()), + clear: method(), + notes: keyed(obj({ title: str() })), + }, +}); + +/** v2: items carry a tag, `add` takes one and returns an object, `clear` is gone, notes have a body. */ +const NotesV2 = evolve(NotesV1, (e) => + e + .state("items", arr(obj({ text: str(), tag: str() }))) + .method("add", obj({ text: str(), tag: str() }), obj({ count: int() })) + .removeMethod("clear") + .keyed("notes", obj({ title: str(), body: str() })), +); + +const Items = defineState("items", { + schema: obj({ items: arr(obj({ text: str(), tag: str() })) }), + version: 1, + initial: () => ({ items: [] }), +}); + +const Notes = defineState("notes", { + schema: obj({ title: str(), body: str() }), + version: 1, + initial: () => ({ title: "", body: "" }), + keyed: true, +}); + +/** Serves `NotesV2`, accepting v1 consumers (`clear` replaced, or refused with `refuse`). */ +function serveV2(h: FacetHost, refuse = false): void { + const clears: string[] = []; + h.serve( + NotesV2, + (_, self) => { + const items = self.state(Items); + const notes = self.state(Notes); + return { + items: self.derive([items], ([value]) => value.items), + notes, + add: async ({ text, tag }: { text: string; tag: string }, ctx: Context) => { + await items.change((draft) => void draft.items.push({ text, tag }), ctx); + await notes.change( + text, + (draft) => { + draft.title = text; + draft.body = tag; + }, + ctx, + ); + return { count: items.value.items.length }; + }, + }; + }, + { + accepts: { + 1: { + state: { items: (current) => current.map(({ text }) => text) }, + methods: { + add: { args: (old) => ({ text: old.text, tag: "v1" }), result: (current) => current.count }, + clear: refuse + ? "refuse" + : { + call: async () => { + clears.push("clear"); + return null; + }, + }, + }, + keyed: { notes: refuse ? "refuse" : (current) => ({ title: current.title }) }, + }, + }, + }, + ); +} + +describe("an older consumer of a provider's accepted version", () => { + it("reads state mapped, with diffs per commit; its calls are mapped forwards and their results back", async () => { + const { telemetry, counts } = await metered(); + const host = createHost({ telemetry, onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h)) }], context); + const old = await host.get(NotesV1, {}, context); + const items = changesOf(await old.items.watch(context)); + expect(await old.add({ text: "a" }, context)).toBe(1); + expect(await old.add({ text: "b" }, context)).toBe(2); + await delivered(); + expect(items).toEqual([ + { kind: "value", value: ["a"], ops: [["p", [], 0, 0, ["a"]]], retired: false }, + { kind: "value", value: ["a", "b"], ops: [["p", [], 1, 0, ["b"]]], retired: false }, + ]); + // Once for the watch's first value, then once per commit: never per consumer or per read. + expect(counts()["chord.contract.adapts"]).toBe(3); + expect(old.items.value).toEqual(["a", "b"]); + expect(counts()["chord.contract.adapts"]).toBe(3); + await host.close(context); + }); + + it("calls the provider's replacement of a removed method; a refused one fails contract_mismatch", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h)) }], context); + const old = await host.get(NotesV1, {}, context); + expect(await old.clear(context)).toBeNull(); + await host.close(context); + }); + + it("is refused a binding whose member the provider refuses: it waits contract_mismatch", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h, true)) }], context); + const old = host.hold(NotesV1, {}, context); + await flushed(); + expect(old.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + }); + expect(await rejection(old.clear(context))).toMatchObject({ code: "contract_mismatch" }); + await host.close(context); + }); + + it("reads keyed members mapped per key: get, watch and a factory's at(key)", async () => { + const View = local("app.view", { version: 1, params: { key: str() }, members: { title: str() } }); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: defineFacet((h) => serveV2(h)) }, + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, ({ key }, self) => { + const note = self.get(NotesV1, {}).notes.at(key); + return { title: self.derive([note], ([value]) => value.title) }; + }), + ), + }, + ], + context, + ); + const old = await host.get(NotesV1, {}, context); + await old.add({ text: "a" }, context); + expect(await old.notes.get("a", context)).toEqual({ title: "a" }); + const watch = (await old.notes.watch("a", context))!; + expect(watch.value).toEqual({ title: "a" }); + const view = await host.get(View, { key: "a" }, context); + expect(view.title.value).toBe("a"); + await host.close(context); + }); +}); + +describe("reload: adapted members follow the provider's topology (fix round 1)", () => { + it("rebuilds a changed-member adapter as a whole-state adapter when a later version removes it", async () => { + const V1 = local("astra.adapt", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ value: int() }))); + const V3 = evolve(V2, (e) => e.removeState("n").addState("m", int())); + const host = createHost({ onError: () => {} }); + const first = defineFacet((h) => + h.serve(V2, () => ({ n: constant({ value: 5 }) }), { accepts: { 1: { state: { n: (n) => n.value } } } }), + ); + const second = defineFacet((h) => + h.serve(V3, () => ({ m: constant(9) }), { accepts: { 1: { state: { n: (s) => s.m } } } }), + ); + await host.load([{ name: "@astra/adapt", facet: first }], context); + const old = await host.get(V1, {}, context); + expect(old.n.value).toBe(5); + await host.reload([{ name: "@astra/adapt", facet: second }], context); + await flushed(); + let actual: unknown; + try { + actual = old.n.value; + } catch (error) { + actual = error; + } + old.release(); + await host.close(context); + expect(actual).toBe(9); + }); + + it("maps existing local watches when the consumer's current version becomes an accepted older one", async () => { + const V1 = local("sol.state", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [{ name: "@sol/state", facet: defineFacet((h) => h.serve(V1, () => ({ x: constant(1) }))) }], + context, + ); + const old = await host.get(V1, {}, context); + const calls = changesOf(await old.x.watch(context)); + await host.reload( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n: 7 }) }), { + accepts: { 1: { state: { x: (value) => value.n } } }, + }), + ), + }, + ], + context, + ); + (await host.get(V2, {}, context)).release(); + await delivered(); + expect(valuesOf(calls)).toEqual([7]); + } finally { + await host.close(context); + } + }); + + it("rebuilds an existing adapted member's dependencies when a later provider removes that member", async () => { + const V1 = local("sol.removed-member", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const V3 = evolve(V2, (e) => e.removeState("x").addState("y", int())); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n: 2 }) }), { + accepts: { 1: { state: { x: (value) => value.n } } }, + }), + ), + }, + ], + context, + ); + const old = await host.get(V1, {}, context); + expect(old.x.value).toBe(2); + await host.reload( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V3, () => ({ y: constant(7) }), { + accepts: { + 1: { state: { x: (state) => state.y } }, + 2: { state: { x: (state) => ({ n: state.y }) } }, + }, + }), + ), + }, + ], + context, + ); + (await host.get(V3, {}, context)).release(); + expect(old.state.value.kind).toBe("live"); + expect(old.x.value).toBe(7); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: adapted members publish one coherent frame (fix round 1)", () => { + it("does not publish the new mapping applied to the old attempt's state", async () => { + const V1 = local("astra.atomic", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", int())); + const facet = (n: number, factor: number) => + defineFacet((h) => + h.serve(V2, () => ({ n: constant(n) }), { accepts: { 1: { state: { n: (v) => v * factor } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@astra/atomic", facet: facet(2, 2) }], context); + const old = await host.get(V1, {}, context); + const watch = await old.n.watch(context); + const changes = changesOf(watch); + await delivered(); + expect(old.n.value).toBe(4); + await host.reload([{ name: "@astra/atomic", facet: facet(3, 3) }], context); + await flushed(); + await delivered(); + const values = changes.map((call) => (call.kind === "value" ? call.value : "failed")); + await watch.stop(); + old.release(); + await host.close(context); + expect(values).toEqual([9]); + }); + + it("never computes a new accepted-version mapping over the old provider shape", async () => { + const V1 = local("sol.mapping-publication", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const V3 = evolve(V2, (e) => e.state("x", obj({ wrapped: obj({ n: int() }) }))); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n: 2 }) }), { + accepts: { 1: { state: { x: (value) => value.n } } }, + }), + ), + }, + ], + context, + ); + const old = await host.get(V1, {}, context); + const calls = changesOf(await old.x.watch(context)); + await host.reload( + [ + { + name: "@sol/state", + facet: defineFacet((h) => + h.serve(V3, () => ({ x: constant({ wrapped: { n: 7 } }) }), { + accepts: { + 1: { state: { x: (value) => value.wrapped.n } }, + 2: { state: { x: (value) => value.wrapped } }, + }, + }), + ), + }, + ], + context, + ); + (await host.get(V3, {}, context)).release(); + await delivered(); + expect(valuesOf(calls)).toEqual([7]); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: an older consumer across a removal (fix round 1)", () => { + it("keeps an older watch open and heals it after the type is served again", async () => { + const V1 = local("sol.older-removal", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const facet = (n: number) => + defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n }) }), { accepts: { 1: { state: { x: (value) => value.n } } } }), + ); + const host = createHost({ onError: () => {} }); + try { + await host.load([{ name: "@sol/older", facet: facet(1) }], context); + const old = await host.get(V1, {}, context); + const watch = await old.x.watch(context); + const calls = changesOf(watch); + await host.reload([], context); + await host.reload([{ name: "@sol/older", facet: facet(7) }], context); + (await host.get(V2, {}, context)).release(); + await delivered(); + // The removal's failure once, then the re-served value, whole. + expect(valuesOf(calls)).toEqual([{ failed: "sol.older-removal is not served here" }, 7]); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: keyed members of an older version follow the provider (fix round 1)", () => { + it("updates an open old-version watch when only its mapping changes", async () => { + const V1 = local("astra.keyed", { version: 1, params: {}, members: { rows: keyed(int()), set: method(int()) } }); + const V2 = evolve(V1, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + keyed: true, + }); + const facet = (factor: number) => + defineFacet((h) => + h.serve( + V2, + (_, self) => { + const rows = self.state(Rows); + return { + rows, + set: async (n, ctx) => { + await rows.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }, + }; + }, + { accepts: { 1: { keyed: { rows: (current) => current.n * factor } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@astra/keyed", facet: facet(2) }], context); + const old = await host.get(V1, {}, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + expect(watch.value).toBe(6); + watch.start(async () => {}); + await host.reload([{ name: "@astra/keyed", facet: facet(4) }], context); + await flushed(); + await old.set(5, context); + await delivered(); + const actual = watch.value; + const fresh = await old.rows.get("a", context); + await watch.stop(); + old.release(); + await host.close(context); + expect({ actual, fresh }).toEqual({ actual: 20, fresh: 20 }); + }); + + it("updates mappings of an existing older watch and factory pin", async () => { + const V1 = local("sol.keyed", { + version: 1, + params: {}, + members: { entries: keyed(obj({ old: int() })), set: method(int()) }, + }); + const V2 = evolve(V1, (e) => e.keyed("entries", obj({ n: int() }))); + const V3 = evolve(V2, (e) => e.keyed("entries", obj({ n: int() }))); + const Doc = defineState("entries", { + version: 1, + schema: obj({ n: int() }), + keyed: true, + initial: () => ({ n: 0 }), + }); + const View = local("sol.keyed.view", { version: 1, params: {}, members: { old: int() } }); + const view = defineFacet((h) => + h.serve(View, (_, self) => { + const pin = self.get(V1, {}).entries.at("k"); + return { old: self.derive([pin], ([value]) => value.old) }; + }), + ); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { + name: "@sol/keyed", + facet: defineFacet((h) => + h.serve( + V2, + (_, self) => { + const entries = self.state(Doc); + return { + entries, + set: async (n: number, ctx: Context) => { + await entries.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }, + }; + }, + { accepts: { 1: { keyed: { entries: (value) => ({ old: value.n }) } } } }, + ), + ), + }, + { name: "@sol/view", facet: view }, + ], + context, + ); + const old = await host.get(V1, {}, context); + const reader = await host.get(View, {}, context); + await old.set(1, context); + const watch = (await old.entries.watch("k", context))!; + const calls = changesOf(watch); + await host.reload( + [ + { + name: "@sol/keyed", + facet: defineFacet((h) => + h.serve( + V3, + (_, self) => { + const entries = self.state(Doc); + return { + entries, + set: async (n: number, ctx: Context) => { + await entries.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }, + }; + }, + { + accepts: { + 1: { keyed: { entries: (value) => ({ old: value.n * 10 }) } }, + 2: { keyed: { entries: (value) => value } }, + }, + }, + ), + ), + }, + { name: "@sol/view", facet: view }, + ], + context, + ); + (await host.get(V3, {}, context)).release(); + await old.set(2, context); + await delivered(); + expect(valuesOf(calls).at(-1)).toEqual({ old: 20 }); + expect(reader.old.value).toBe(20); + } finally { + await host.close(context); + } + }); +}); + +describe("adapted members' lifetime (fix round 1)", () => { + const adapted = instancesOf(AdaptedMembers, "member"); + + it("drops an older version's adapted members when its last consumer leaves, while the object lives on", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serveV2(h)) }], context); + const current = await host.get(NotesV2, {}, context); + const from = adapted.length; + const old = await host.get(NotesV1, {}, context); + expect(old.items.value).toEqual([]); + expect(adapted.length).toBeGreaterThan(from); + // The older version's: those of the version shown stay while the object lives (the mappings read them). + const older = adapted.slice(from).filter((ref) => ref.deref()?.version === 1); + expect(older.length).toBeGreaterThan(0); + old.release(); + await flushed(); + await expectCollected(...older); + expect(current.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("reload: keyed members unchanged at a consumer's version follow a later mapping (fix round 1)", () => { + const RowsV1 = local("app.rows", { + version: 1, + params: {}, + members: { rows: keyed(obj({ n: int() })), set: method(int()) }, + }); + /** v2 adds a member: `rows` is unchanged for v1 consumers. */ + const RowsV2 = evolve(RowsV1, (e) => e.addState("extra", int())); + /** v3 changes `rows` (same shape): v1 and v2 consumers read it mapped. */ + const RowsV3 = evolve(RowsV2, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 0 }), keyed: true }); + const View = local("app.rows.view", { version: 1, params: {}, members: { n: int() } }); + const ten = (value: { readonly n: number }) => ({ n: value.n * 10 }); + + /** Serves `contract` with its keyed rows, `set` writing row "k", and `accepts`. */ + function rows(contract: typeof RowsV1 | typeof RowsV2 | typeof RowsV3, accepts: object) { + return defineFacet((h) => + h.serve( + contract as typeof RowsV3, + (_, self) => { + const docs = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await docs.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return (contract === RowsV1 ? { rows: docs, set } : { rows: docs, set, extra: constant(0) }) as never; + }, + { accepts } as never, + ), + ); + } + + /** A v1 consumer's factory pin of row "k". */ + const view = defineFacet((h) => + h.serve(View, (_, self) => { + const pin = self.get(RowsV1, {}).rows.at("k"); + return { n: self.derive([pin], ([value]) => value.n) }; + }), + ); + + /** Loads `first`, watches and pins row "k" as a v1 consumer, reloads to `second`, sets 2 and reads all three. */ + async function across(first: ReturnType, second: ReturnType) { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@app/rows", facet: first }, + { name: "@app/view", facet: view }, + ], + context, + ); + const old = await host.get(RowsV1, {}, context); + const reader = await host.get(View, {}, context); + await old.set(1, context); + const watch = (await old.rows.watch("k", context))!; + watch.start(async () => {}); + await host.reload( + [ + { name: "@app/rows", facet: second }, + { name: "@app/view", facet: view }, + ], + context, + ); + await flushed(); + await old.set(2, context); + await delivered(); + const result = { watch: watch.value, get: await old.rows.get("k", context), pin: reader.n.value }; + await watch.stop(); + old.release(); + reader.release(); + await host.close(context); + return result; + } + + it("re-maps a key watched and pinned at the provider's current version once a later provider maps it", async () => { + const result = await across( + rows(RowsV1, {}), + rows(RowsV3, { 1: { keyed: { rows: ten } }, 2: { keyed: { rows: ten } } }), + ); + expect(result).toEqual({ watch: { n: 20 }, get: { n: 20 }, pin: 20 }); + }); + + it("re-maps a key of an older version its provider left unchanged once a later provider maps it", async () => { + const result = await across( + rows(RowsV2, { 1: {} }), + rows(RowsV3, { 1: { keyed: { rows: ten } }, 2: { keyed: { rows: ten } } }), + ); + expect(result).toEqual({ watch: { n: 20 }, get: { n: 20 }, pin: 20 }); + }); +}); + +describe("reload: keyed sources of an older version through a removal (fix round 2)", () => { + const V1 = local("app.keyed.removal", { + version: 1, + params: {}, + members: { rows: keyed(int()), set: method(int()) }, + }); + const V2 = evolve(V1, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 0 }), keyed: true }); + const View = local("app.keyed.removal.view", { version: 1, params: {}, members: { n: int() } }); + const keyMappings = instancesOf(AdaptedMembers, "key"); + + /** Serves `V2`, mapping its rows to v1 times `factor`; `set` writes row "a". */ + const rows = (factor: number) => + defineFacet((h) => + h.serve( + V2, + (_, self) => { + const docs = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await docs.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { rows: docs, set }; + }, + { accepts: { 1: { keyed: { rows: (current) => current.n * factor } } } }, + ), + ); + + /** A v1 consumer's factory pin of row "a". */ + const view = defineFacet((h) => + h.serve(View, (_, self) => { + const pin = self.get(V1, {}).rows.at("a"); + return { n: self.derive([pin], ([value]) => value) }; + }), + ); + + it("re-maps an open older keyed watch and a factory pin through the re-served provider (Astra, Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@app/rows", facet: rows(2) }, + { name: "@app/view", facet: view }, + ], + context, + ); + const old = await host.get(V1, {}, context); + const reader = await host.get(View, {}, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + const calls = changesOf(watch); + expect([watch.value, reader.n.value]).toEqual([6, 6]); + await host.reload([{ name: "@app/view", facet: view }], context); + await flushed(); + // Frozen while the type is gone: the last mapped value. + expect(watch.value).toBe(6); + await host.reload( + [ + { name: "@app/rows", facet: rows(4) }, + { name: "@app/view", facet: view }, + ], + context, + ); + await flushed(); + await old.set(5, context); + await delivered(); + expect({ watch: watch.value, get: await old.rows.get("a", context), pin: reader.n.value }).toEqual({ + watch: 20, + get: 20, + pin: 20, + }); + expect(valuesOf(calls).at(-1)).toBe(20); + await watch.stop(); + old.release(); + reader.release(); + await host.close(context); + }); + + it("keeps no removed instance's mappings once the reference moved, its watch open (decision: references only)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const from = keyMappings.length; + const old = await host.get(V1, {}, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + watch.start(async () => {}); + // The removed instance's mapping of the key: nothing of it outlives it through the reference. + const removed = keyMappings.slice(from); + expect(removed.length).toBeGreaterThan(0); + await host.reload([], context); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + // The re-served object's documents start over (no store here). + await old.set(3, context); + await delivered(); + expect(watch.value).toBe(12); + await expectCollected(...removed); + await watch.stop(); + old.release(); + await host.close(context); + }); +}); + +describe("reload: a version refusal keeps the waiting reference's last values (fix round 2)", () => { + const V1 = local("app.version.return", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ value: int() }))); + const accepted = (value: number) => + defineFacet((h) => + h.serve(V2, () => ({ n: constant({ value }) }), { accepts: { 1: { state: { n: (v) => v.value } } } }), + ); + const refused = defineFacet((h) => h.serve(V2, () => ({ n: constant({ value: 2 }) }))); + + it("keeps an older watch open while its version is refused and heals it when it is accepted again (Astra)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/version", facet: accepted(4) }], context); + const old = await host.get(V1, {}, context); + const watch = await old.n.watch(context); + const calls = changesOf(watch); + let closed = false; + void watch.closed.then(() => { + closed = true; + }); + await host.reload([{ name: "@app/version", facet: refused }], context); + await delivered(); + expect(old.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // Its last mapped value, read directly too. + expect([watch.value, old.n.value]).toEqual([4, 4]); + await host.reload([{ name: "@app/version", facet: accepted(9) }], context); + await delivered(); + expect({ closed, watch: watch.value, read: old.n.value, last: valuesOf(calls).at(-1) }).toEqual({ + closed: false, + watch: 9, + read: 9, + last: 9, + }); + old.release(); + await host.close(context); + }); + + it("heals a watch of the previous current version once a later provider accepts it (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/version", facet: defineFacet((h) => h.serve(V1, () => ({ n: constant(1) }))) }], + context, + ); + const old = await host.get(V1, {}, context); + const calls = changesOf(await old.n.watch(context)); + await host.reload([{ name: "@app/version", facet: refused }], context); + (await host.get(V2, {}, context)).release(); + expect(old.state.value.kind).toBe("failed"); + await host.reload([{ name: "@app/version", facet: accepted(3) }], context); + (await host.get(V2, {}, context)).release(); + await delivered(); + expect(old.state.value.kind).toBe("live"); + expect(valuesOf(calls).at(-1)).toBe(3); + old.release(); + await host.close(context); + }); +}); + +describe("reload: watches of one version across a rollback (fix round 2)", () => { + it("keeps both generations of v1 watches after v1 to v2 to v1 to v2 (Sol)", async () => { + const V1 = local("app.rollback", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const v1 = (n: number) => defineFacet((h) => h.serve(V1, () => ({ x: constant(n) }))); + const v2 = (n: number) => + defineFacet((h) => + h.serve(V2, () => ({ x: constant({ n }) }), { accepts: { 1: { state: { x: (value) => value.n } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rollback", facet: v1(1) }], context); + const first = await host.get(V1, {}, context); + const firstCalls = changesOf(await first.x.watch(context)); + await host.reload([{ name: "@app/rollback", facet: v2(2) }], context); + (await host.get(V2, {}, context)).release(); + await host.reload([{ name: "@app/rollback", facet: v1(3) }], context); + const second = await host.get(V1, {}, context); + const secondCalls = changesOf(await second.x.watch(context)); + await host.reload([{ name: "@app/rollback", facet: v2(4) }], context); + (await host.get(V2, {}, context)).release(); + await delivered(); + expect(valuesOf(firstCalls)).toEqual([2, 3, 4]); + expect(valuesOf(secondCalls)).toEqual([4]); + expect(second.x.value).toBe(4); + first.release(); + second.release(); + await host.close(context); + }); +}); + +describe("reload: an older consumer's reads while its type is removed (fix round 2)", () => { + const V1 = local("app.removed.value", { version: 1, params: {}, members: { n: int(), gone: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ value: int() })).removeState("gone")); + const facet = defineFacet((h) => + h.serve(V2, () => ({ n: constant({ value: 5 }) }), { + accepts: { 1: { state: { n: (v) => v.value, gone: (state) => state.n.value + 1 } } }, + }), + ); + + it("reads its last mapped value, not the provider's raw shape, a member read before or not (Astra)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/removed", facet }], context); + const old = await host.get(V1, {}, context); + expect(old.n.value).toBe(5); + await host.reload([], context); + // `gone` was never read before the removal: the current version has no such member. + expect({ n: old.n.value, gone: old.gone.value }).toEqual({ n: 5, gone: 6 }); + old.release(); + await host.close(context); + }); +}); + +describe("reload: an older consumer's first read during a parked swap (fix round 2)", () => { + it("maps with the provider whose attempt is shown until the new one goes live (Sol)", async () => { + const V1 = local("app.lazy", { version: 1, params: {}, members: { x: int() } }); + const V2 = evolve(V1, (e) => e.state("x", obj({ n: int() }))); + const Missing = local("app.lazy.missing", { version: 1, params: {}, members: {} }); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/lazy", facet: defineFacet((h) => h.serve(V1, () => ({ x: constant(1) }))) }], + context, + ); + const old = await host.get(V1, {}, context); + // Nothing read before the swap: the first read comes while the new attempt parks on a missing dependency. + const v2 = defineFacet((h) => + h.serve( + V2, + (_, self) => { + self.get(Missing, {}); + return { x: constant({ n: 7 }) }; + }, + { accepts: { 1: { state: { x: (value) => value.n } } } }, + ), + ); + await host.reload([{ name: "@app/lazy", facet: v2 }], context); + await delivered(); + expect(old.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(old.x.value).toBe(1); + const calls = changesOf(await old.x.watch(context)); + await host.reload( + [ + { name: "@app/lazy", facet: v2 }, + { name: "@app/missing", facet: defineFacet((h) => h.serve(Missing, () => ({}))) }, + ], + context, + ); + await delivered(); + expect(old.state.value).toEqual({ kind: "live" }); + expect([old.x.value, valuesOf(calls).at(-1)]).toEqual([7, 7]); + old.release(); + await host.close(context); + }); +}); + +describe("reload: references keep their last values through a removal, a refusal and an end while they wait (fix round 3)", () => { + const V1 = local("app.waiting", { + version: 1, + params: { id: int() }, + members: { rows: keyed(int()), set: method(int()) }, + }); + const V2 = evolve(V1, (e) => e.keyed("rows", obj({ n: int() }))); + const Rows = defineState("rows", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 0 }), keyed: true }); + const keyMappings = instancesOf(AdaptedMembers, "key"); + + /** Serves `V2`, `set` writing row "a"; mapping v1's rows times `factor`, or refusing v1 without one. */ + const rows = (factor?: number) => + defineFacet((h) => + h.serve( + V2, + (_, self) => { + const docs = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await docs.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { rows: docs, set }; + }, + factor === undefined ? {} : { accepts: { 1: { keyed: { rows: (current) => current.n * factor } } } }, + ), + ); + + /** A v1 reference to object `id` with row "a" at 3 and an open, started watch of it. */ + async function watched(host: Host, id: number) { + const old = await host.get(V1, { id }, context); + await old.set(3, context); + const watch = (await old.rows.watch("a", context))!; + const calls = changesOf(watch); + let closed = false; + void watch.closed.then(() => { + closed = true; + }); + return { old, watch, calls, closed: () => closed }; + } + + it("keeps an older keyed watch when the refusal idles its object, which ends (Astra)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const { old, watch, closed } = await watched(host, 1); + expect(watch.value).toBe(6); + // v1 refused: its only holder waits, so the object (with params) ends; its pinned keys wait too. + await host.reload([{ name: "@app/rows", facet: rows() }], context); + await flushed(); + expect(old.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + await old.set(5, context); + await delivered(); + expect({ closed: closed(), watch: watch.value, get: await old.rows.get("a", context) }).toEqual({ + closed: false, + watch: 20, + get: 20, + }); + old.release(); + await host.close(context); + }); + + it("keeps an older keyed watch through a refusal and then a removal of its type (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const { old, calls, closed } = await watched(host, 1); + const current = await host.get(V2, { id: 1 }, context); + await host.reload([{ name: "@app/rows", facet: rows() }], context); + await delivered(); + expect(old.state.value.kind).toBe("failed"); + current.release(); + await host.reload([], context); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + await old.set(3, context); + await delivered(); + expect([closed(), valuesOf(calls).at(-1)]).toEqual([false, 12]); + old.release(); + await host.close(context); + }); + + it("keeps no removed instance's mappings while its keyed watch stays open (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rows", facet: rows(2) }], context); + const from = keyMappings.length; + const { old, watch } = await watched(host, 1); + const removed = keyMappings.slice(from); + expect(removed.length).toBeGreaterThan(0); + await host.reload([], context); + await host.reload([{ name: "@app/rows", facet: rows(4) }], context); + await old.set(3, context); + await delivered(); + expect(watch.value).toBe(12); + // The watch is still open: what it follows now is the re-served object's instance. + await expectCollected(...removed); + old.release(); + await host.close(context); + }); + + it("reads an older member never read before at its last mapped value after a refusal (Astra)", async () => { + const State1 = local("app.waiting.state", { version: 1, params: {}, members: { n: int() } }); + const State2 = evolve(State1, (e) => e.state("n", obj({ value: int() }))); + const serve = (value: number, accepts: boolean) => + defineFacet((h) => + h.serve( + State2, + () => ({ n: constant({ value }) }), + accepts ? { accepts: { 1: { state: { n: (v) => v.value } } } } : {}, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/state", facet: serve(4, true) }], context); + const old = await host.get(State1, {}, context); + await host.reload([{ name: "@app/state", facet: serve(9, false) }], context); + await flushed(); + expect(old.n.value).toBe(4); + old.release(); + await host.close(context); + }); + + it("follows a newer consumer through two rollbacks, its last value kept while refused (Sol)", async () => { + const X1 = local("app.waiting.rollback", { version: 1, params: {}, members: { x: int() } }); + const X2 = evolve(X1, (e) => e.state("x", obj({ n: int() }))); + const v1 = (n: number) => defineFacet((h) => h.serve(X1, () => ({ x: constant(n) }))); + const v2 = (n: number) => + defineFacet((h) => + h.serve(X2, () => ({ x: constant({ n }) }), { accepts: { 1: { state: { x: (value) => value.n } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/rollback", facet: v2(1) }], context); + const held = await host.get(X2, {}, context); + const calls = changesOf(await held.x.watch(context)); + for (const [back, ahead] of [ + [2, 3], + [4, 5], + ]) { + // A v1 provider cannot serve the v2 consumer: it waits with its source, then takes it back. + await host.reload([{ name: "@app/rollback", facet: v1(back!) }], context); + await delivered(); + expect(held.state.value.kind).toBe("failed"); + await host.reload([{ name: "@app/rollback", facet: v2(ahead!) }], context); + (await host.get(X2, {}, context)).release(); + await delivered(); + } + // Each refusal's failure once, then each return's value, whole. + const refused = { failed: "app.waiting.rollback v2 is not served here" }; + expect([held.x.value, valuesOf(calls)]).toEqual([{ n: 5 }, [refused, { n: 3 }, refused, { n: 5 }]]); + held.release(); + await host.close(context); + }); +}); + +describe("reload: a released reference's last values go with it while another version waits (fix round 4)", () => { + it("drops a removed version's mapped value when its last request leaves while another version waits (Astra)", async () => { + const H1 = local("app.waiting.versions", { + version: 1, + params: { id: int() }, + members: { n: obj({ mapped: int() }) }, + }); + const H2 = evolve(H1, (e) => e.state("n", obj({ raw: int() }))); + const facet = defineFacet((h) => + h.serve(H2, () => ({ n: constant({ raw: 7 }) }), { + accepts: { 1: { state: { n: (v) => ({ mapped: v.raw }) } } }, + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/versions", facet }], context); + const older = await host.get(H1, { id: 1 }, context); + const newer = await host.get(H2, { id: 1 }, context); + const mapped = new WeakRef(older.n.value); + // Both versions wait with their last values; v1's reference is released, v2's stays. + await host.reload([], context); + older.release(); + await flushed(); + await expectCollected(mapped); + newer.release(); + await host.close(context); + }); + + it("drops a refused version's snapshot when its last request leaves while an older one waits (Sol)", async () => { + const G1 = local("app.waiting.chain", { version: 1, params: {}, members: { x: obj({ n: int() }) } }); + const G2 = evolve(G1, (e) => e.state("x", obj({ n: int() }))); + const G3 = evolve(G2, (e) => e.state("x", obj({ n: int() }))); + const serve = (contract: typeof G1 | typeof G2 | typeof G3, n: number) => + defineFacet((h) => h.serve(contract as typeof G3, () => ({ x: constant({ n }) }))); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/chain", facet: serve(G1, 1) }], context); + const first = await host.get(G1, {}, context); + await host.reload([{ name: "@app/chain", facet: serve(G2, 2) }], context); + const second = await host.get(G2, {}, context); + const snapshot = new WeakRef(second.x.value); + // v3 accepts neither: both wait; v2's last request leaves, v1's stays. + await host.reload([{ name: "@app/chain", facet: serve(G3, 3) }], context); + (await host.get(G3, {}, context)).release(); + second.release(); + await delivered(); + expect(first.state.value.kind).toBe("failed"); + await expectCollected(snapshot); + first.release(); + await host.close(context); + }); +}); + +describe("reload: stopping a keyed watch drops its pin and last value while its reference waits (fix round 4)", () => { + it("drops a stopped key's document while its reference still waits (Sol)", async () => { + const K1 = local("app.waiting.key", { + version: 1, + params: {}, + members: { rows: keyed(obj({ n: int() })), extra: int(), set: method(int()) }, + }); + const K2 = evolve(K1, (e) => e.state("extra", int())); + const Doc = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 1 }), + keyed: true, + }); + const facet = defineFacet((h) => + h.serve( + K2, + (_, self) => { + const rows = self.state(Doc); + const set = async (n: number, ctx: Context) => { + await rows.change( + "k", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { rows, extra: constant(0), set }; + }, + { accepts: { 1: { state: { extra: (value) => value } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/key", facet }], context); + const ref = await host.get(K1, {}, context); + await ref.set(3, context); + // A watch of the key while the type is removed (the reference waits, its key with its last value), then + // stopped; the watch itself goes out of scope here. + const watched = async () => { + const watch = (await ref.rows.watch("k", context))!; + watch.start(async () => {}); + const value = new WeakRef(watch.value); + await host.reload([], context); + await watch.stop(); + return value; + }; + const document = await watched(); + expect(ref.state.value.kind).toBe("failed"); + await expectCollected(document); + ref.release(); + await host.close(context); + }); +}); + +describe("reload: a reference that never had values reads none during a parked swap (fix round 4)", () => { + it("has a newer reference read nothing during a parked swap, its watch waiting, then the new instance's values (Sol)", async () => { + const N1 = local("app.unshown", { version: 1, params: {}, members: { x: int() } }); + const N2 = evolve(N1, (e) => e.state("x", obj({ n: int() }))); + const Missing = local("app.unshown.missing", { version: 1, params: {}, members: {} }); + const v2 = defineFacet((h) => + h.serve( + N2, + (_, self) => { + self.get(Missing, {}); + return { x: constant({ n: 7 }) }; + }, + { accepts: { 1: { state: { x: (value) => value.n } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/unshown", facet: defineFacet((h) => h.serve(N1, () => ({ x: constant(1) }))) }], + context, + ); + const old = await host.get(N1, {}, context); + await host.reload([{ name: "@app/unshown", facet: v2 }], context); + const newer = host.hold(N2, {}, context); + expect(newer.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // The swap parks: the shown values are v1's; a v2 reference never had values, never v1's shape: its read throws + // the parked swap's failure. + expect(old.x.value).toBe(1); + expect(() => newer.x.value).toThrow(expect.objectContaining({ code: "unavailable" })); + // A watch waits for its first values through the failure (Opus G3). + const waiting = newer.x.watch(context); + await host.reload( + [ + { name: "@app/unshown", facet: v2 }, + { name: "@app/missing", facet: defineFacet((h) => h.serve(Missing, () => ({}))) }, + ], + context, + ); + await delivered(); + expect(newer.state.value).toEqual({ kind: "live" }); + expect([newer.x.value, old.x.value]).toEqual([{ n: 7 }, 7]); + expect([(await waiting).value, (await newer.x.watch(context)).value]).toEqual([{ n: 7 }, { n: 7 }]); + newer.release(); + old.release(); + await host.close(context); + }); +}); + +describe("reload: a reference reads its last value while its version's return parks (fix round 5)", () => { + it("reads the last value its surviving watch shows after v2 to v1 to a parked v2 (Sol)", async () => { + const R1 = local("app.return", { version: 1, params: {}, members: { x: int() } }); + const R2 = evolve(R1, (e) => e.state("x", obj({ n: int() }))); + const Missing = local("app.return.missing", { version: 1, params: {}, members: {} }); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/return", facet: defineFacet((h) => h.serve(R2, () => ({ x: constant({ n: 7 }) }))) }], + context, + ); + const ref = await host.get(R2, {}, context); + const watch = await ref.x.watch(context); + watch.start(async () => {}); + // A v1 provider cannot serve v2: the reference waits with its last value. + await host.reload( + [{ name: "@app/return", facet: defineFacet((h) => h.serve(R1, () => ({ x: constant(1) }))) }], + context, + ); + expect(ref.x.value).toEqual({ n: 7 }); + // v2 is served again but its swap parks: the sources taken back are v2's own, shown before. + const parked = defineFacet((h) => + h.serve(R2, (_, self) => { + self.get(Missing, {}); + return { x: constant({ n: 9 }) }; + }), + ); + await host.reload([{ name: "@app/return", facet: parked }], context); + await delivered(); + expect([watch.value, ref.x.value, (await ref.x.watch(context)).value]).toEqual([{ n: 7 }, { n: 7 }, { n: 7 }]); + ref.release(); + await host.close(context); + }); +}); + +describe("reload: one rule for reads and watches across a new object (fix round 6)", () => { + const Type = local("app.successor", { version: 1, params: {}, members: { x: int() } }); + const Missing = local("app.successor.missing", { version: 1, params: {}, members: {} }); + + it("reads the last value its watch shows while the re-served object parks (Sol)", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/successor", facet: defineFacet((h) => h.serve(Type, () => ({ x: constant(7) }))) }], + context, + ); + const ref = await host.get(Type, {}, context); + const watch = await ref.x.watch(context); + watch.start(async () => {}); + await host.reload([], context); + expect(ref.x.value).toBe(7); + // Re-served: a new object, which parks before any instance of it went live: the reference keeps its last value. + const parked = defineFacet((h) => + h.serve(Type, (_, self) => { + self.get(Missing, {}); + return { x: constant(9) }; + }), + ); + await host.reload([{ name: "@app/successor", facet: parked }], context); + await delivered(); + expect(ref.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect([watch.value, ref.x.value]).toEqual([7, 7]); + ref.release(); + await host.close(context); + }); + + it("opens a new watch at its last value while the re-served object's native opens (Sol)", async () => { + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/successor", facet: defineFacet((h) => h.serve(Type, () => ({ x: constant(7) }))) }], + context, + ); + const ref = await host.get(Type, {}, context); + await host.reload([], context); + const opening = defineFacet((h) => + h.serve(Type, (_, self) => { + self.open( + async () => gate.promise, + () => {}, + ); + return { x: constant(9) }; + }), + ); + const reloading = host.reload([{ name: "@app/successor", facet: opening }], context); + await delivered(); + expect(ref.state.value.kind).toBe("loading"); + expect(ref.x.value).toBe(7); + // At once, not after hydration: the source of its version exists. + const watch = await ref.x.watch(context); + expect(watch.value).toBe(7); + const calls = changesOf(watch); + gate.resolve(1); + await reloading; + await delivered(); + expect(valuesOf(calls).at(-1)).toBe(9); + ref.release(); + await host.close(context); + }); +}); + +describe("reload: an older version only a pending provider maps has no values yet (fix round 7)", () => { + const A1 = local("app.pending.accept", { version: 1, params: {}, members: { x: int() } }); + const A2 = evolve(A1, (e) => e.state("x", obj({ n: int() }))); + const A3 = evolve(A2, (e) => e.state("x", obj({ wrapped: obj({ n: int() }) }))); + + /** v2 shown (no accepts for v1), a reload to v3 (accepting v1 and v2) gated on a native; a new v1 hold, loading. */ + async function pending() { + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "@app/accept", facet: defineFacet((h) => h.serve(A2, () => ({ x: constant({ n: 2 }) }))) }], + context, + ); + const v3 = defineFacet((h) => + h.serve( + A3, + (_, self) => { + self.open( + async () => gate.promise, + () => {}, + ); + return { x: constant({ wrapped: { n: 7 } }) }; + }, + { + accepts: { + 1: { state: { x: (value) => value.wrapped.n } }, + 2: { state: { x: (value) => value.wrapped } }, + }, + }, + ), + ); + const reloading = host.reload([{ name: "@app/accept", facet: v3 }], context); + await delivered(); + return { host, ref: host.hold(A1, {}, context), gate, reloading }; + } + + it("reads not_ready rather than asking the shown provider for a mapping it lacks (Sol)", async () => { + const { host, ref, gate, reloading } = await pending(); + expect(ref.state.value.kind).toBe("loading"); + expect(() => ref.x.value).toThrow(expect.objectContaining({ code: "not_ready" })); + gate.resolve(1); + await reloading; + await delivered(); + expect(ref.x.value).toBe(7); + ref.release(); + await host.close(context); + }); + + it("opens a watch once the accepting provider is live, not before (Sol)", async () => { + const { host, ref, gate, reloading } = await pending(); + let opened = false; + const opening = ref.x.watch(context).then((watch) => { + opened = true; + return watch; + }); + await delivered(); + expect(opened).toBe(false); + gate.resolve(1); + await reloading; + expect((await opening).value).toBe(7); + ref.release(); + await host.close(context); + }); +}); + +describe('an older consumer keeps nothing of its instance\'s mapping (decision "References are the only indirection")', () => { + const mappings = instancesOf(AdaptedMembers, "member"); + + it("lets an ended instance's mapping derives go while the older reference waits with an open watch", async () => { + const M1 = local("app.mapped.gc", { version: 1, params: {}, members: { n: int() } }); + const M2 = evolve(M1, (e) => e.state("n", obj({ value: int() }))); + const facet = defineFacet((h) => + h.serve(M2, () => ({ n: constant({ value: 4 }) }), { accepts: { 1: { state: { n: (v) => v.value } } } }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/mapped", facet }], context); + const from = mappings.length; + const old = await host.get(M1, {}, context); + const calls = changesOf(await old.n.watch(context)); + const used = mappings.slice(from); + expect(used.length).toBeGreaterThan(0); + await host.reload([], context); + await delivered(); + // Its last mapped value and the failure once; the instance's mapping went with it. + expect([old.n.value, valuesOf(calls)]).toEqual([4, [{ failed: "app.mapped.gc is not served here" }]]); + await expectCollected(...used); + old.release(); + await host.close(context); + }); +}); + +describe("an instance's mapping detaches only what it made (final review)", () => { + it("keeps an unchanged member, the instance's own derive, when the last older reference leaves (Astra)", async () => { + const D1 = local("app.shared-derive", { version: 1, params: {}, members: { n: int(), set: method(int()) } }); + const D2 = evolve(D1, (e) => e.addState("extra", int())); + const Counter = defineState("counter", { version: 1, schema: obj({ n: int() }), initial: () => ({ n: 1 }) }); + const facet = defineFacet((h) => + h.serve( + D2, + (_, self) => { + const counter = self.state(Counter); + const set = async (n: number, ctx: Context) => { + await counter.change((draft) => { + draft.n = n; + }, ctx); + return null; + }; + return { n: self.derive([counter], ([c]) => c.n * 2), extra: constant(0), set }; + }, + { accepts: { 1: {} } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/derive", facet }], context); + const newer = await host.get(D2, {}, context); + const older = await host.get(D1, {}, context); + // v1 reads `n` unchanged: the instance's own derive, not a mapping. + expect(older.n.value).toBe(2); + const calls = changesOf(await newer.n.watch(context)); + older.release(); + await newer.set(3, context); + await delivered(); + expect([newer.n.value, valuesOf(calls)]).toEqual([6, [6]]); + newer.release(); + await host.close(context); + }); +}); + +describe("keyed mappings per member (final review)", () => { + it("maps one document exposed as two keyed members through each member's own mapping (Astra)", async () => { + const K1 = local("app.keyed-alias", { + version: 1, + params: {}, + members: { first: keyed(int()), second: keyed(int()), set: method(int()) }, + }); + const K2 = evolve(K1, (e) => e.keyed("first", obj({ n: int() })).keyed("second", obj({ n: int() }))); + const Rows = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + keyed: true, + }); + const facet = defineFacet((h) => + h.serve( + K2, + (_, self) => { + const rows = self.state(Rows); + const set = async (n: number, ctx: Context) => { + await rows.change( + "a", + (draft) => { + draft.n = n; + }, + ctx, + ); + return null; + }; + return { first: rows, second: rows, set }; + }, + { accepts: { 1: { keyed: { first: (value) => value.n, second: (value) => value.n * 10 } } } }, + ), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/keyed-alias", facet }], context); + const ref = await host.get(K1, {}, context); + await ref.set(5, context); + const first = (await ref.first.watch("a", context))!; + const second = (await ref.second.watch("a", context))!; + expect({ first: first.value, second: second.value, read: await ref.second.get("a", context) }).toEqual({ + first: 5, + second: 50, + read: 50, + }); + await first.stop(); + await second.stop(); + ref.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/attempt.test.ts b/packages/chord/test/next/host/attempt.test.ts index 867502320..9f7e7e46a 100644 --- a/packages/chord/test/next/host/attempt.test.ts +++ b/packages/chord/test/next/host/attempt.test.ts @@ -1,8 +1,8 @@ import { createHook } from "node:async_hooks"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, expectTypeOf, it, vi } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; import { arr, int, local, method, obj, opt, remote, str } from "../../../src/next/contracts/index.ts"; -import { broken, isChordError } from "../../../src/next/data/index.ts"; +import { broken, ChordError, isChordError } from "../../../src/next/data/index.ts"; import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; import { createHost, @@ -12,6 +12,8 @@ import { type Self, type StoreSelf, } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; import type { ChordStore } from "../../../src/next/store/store.ts"; @@ -20,7 +22,21 @@ import { TableChanges } from "../../../src/next/store/table-changes.ts"; import { evolveTable } from "../../../src/next/store/table-evolve.ts"; import { TableHandle } from "../../../src/next/store/table-handle.ts"; import { prepareTables } from "../../../src/next/store/table-migration.ts"; -import { failures, fakeTimers, flushed, loaded, memoryStore, metered, rejection, tableOwner } from "../support.ts"; +import { + failLive, + failures, + fakeTimers, + flushed, + instancesOf, + loaded, + memoryStore, + metered, + rejection, + Server, + tableOwner, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); const List = defineState("list", { schema: obj({ items: arr(str()) }), version: 1, initial: () => ({ items: [] }) }); const Page = local("app.page", { version: 1, params: { id: int() }, members: { label: str(), slow: method() } }); @@ -425,6 +441,36 @@ describe("declarations", () => { () => {}, ), ).toThrow("self.open is valid only in the factory"); + const Names = defineRegistry("app.names", { version: 1 }); + expect(() => later!.registry(Names)).toThrow("self.registry is valid only in the factory"); + expect(() => later!.writable(1)).toThrow("self.writable is valid only in the factory"); + expect(() => later!.retryOn()).toThrow("self.retryOn is valid only in the factory"); + }); + + it("contribute only in host-lifetime objects (R21): `Self` has no `contribute`, a declaration error at runtime", async () => { + expectTypeOf().not.toHaveProperty("contribute"); + expectTypeOf().toHaveProperty("contribute"); + const Names = defineRegistry("app.names", { version: 1 }); + const Store = local("app.store", { version: 1, params: {}, members: {} }); + const { host } = await loaded((h) => { + h.serve(Names, { compose: (table) => table.length }); + h.serve(Page, (_, self) => { + (self as unknown as StoreSelf).contribute(Names, "page"); + return { label: constant(""), slow: async () => null }; + }); + h.serve(Store, (_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + return {}; + }); + }); + await expect(host.get(Page, { id: 1 }, context)).rejects.toThrow( + "app.page(1): self.contribute is valid only in a local type with empty params", + ); + await expect(host.get(Store, {}, context)).rejects.toThrow( + "app.store: registry app.names is contributed to twice", + ); + await host.close(context); }); it("refuse a document definition declared twice", async () => { @@ -534,6 +580,7 @@ describe("remote variants", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { n: int() } } }, }); const owner = { kind: "owner" } as const; @@ -541,7 +588,11 @@ describe("remote variants", () => { /** A host acting as `owner` whose factory for `Two` returns `implementation`. */ async function two(implementation: unknown) { const errors: HostError[] = []; - const host = createHost({ principal: owner, onError: (error) => errors.push(error) }); + const host = createHost({ + address: Server(), + principal: owner, + onError: (error) => errors.push(error), + }); const facet = defineFacet((h) => h.serve(Two, () => implementation as never)); await host.load([{ name: "@test/app", facet }], context); return { host, errors }; @@ -846,3 +897,97 @@ describe("eager migrations at activation (slice 6a §9)", () => { await store.close(context); }); }); + +describe("parked attempts (dependency model §3)", () => { + fakeTimers(); + const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + /** A view with params: no host lifetime, so `load` does not wait for it. */ + const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: { n: int() } }); + + it("run no timer while parked but parked_long's: no dependency deadline, no backoff, the factory run once", async () => { + const factories = vi.fn(); + const { host, errors } = await loaded((h) => + h.serve(View, (_, self) => { + factories(); + return { n: self.get(Missing, {}).n }; + }), + ); + const view = host.hold(View, {}, context); + await flushed(); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.missing: app.missing is not served here" } }, + retryAt: undefined, + }); + // `parked_long`'s one timer fired (once, after readyTimeout); nothing else runs. + expect(vi.getTimerCount()).toBe(0); + expect([factories.mock.calls.length, errors]).toEqual([ + 1, + [{ kind: "parked_long", object: "app.view", waitingFor: "app.missing", state: "failed", since: 0 }], + ]); + view.release(); + }); + + it("pause the readyTimeout deadline while parked: the own steps get the budget left", async () => { + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const { host, errors } = await loaded( + (h) => { + h.serve(Slow, (_, self) => { + self.open( + () => new Promise((resolve) => setTimeout(resolve, 5_000)), + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(Viewer, (_, self) => { + const slow = self.get(Slow, { id: 1 }); + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { n: slow.n }; + }); + }, + { readyTimeout: 10_000 }, + ); + const view = host.hold(Viewer, { id: 1 }, context); + // Parked 5 s on its dependency; then its own open hangs: it fails 10 s after the boundary, not 10 s after start. + await vi.advanceTimersByTimeAsync(5_000 + 9_999); + expect(failures(errors)).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(failures(errors)).toEqual(["app.viewer(1): transient"]); + view.release(); + }); + + it("go on past the dependency check: a dependency leaving live during the own steps changes nothing (R18)", async () => { + const Dependency = local("app.dependency", { version: 1, params: {}, members: { n: int() } }); + const gate = Promise.withResolvers(); + let dependencySelf: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Dependency, (_, self) => { + dependencySelf = self; + return { n: constant(1) }; + }); + h.serve(Viewer, (_, self) => { + const dependency = self.get(Dependency, {}); + self.open( + () => gate.promise, + () => {}, + ); + return { n: dependency.n }; + }); + }); + const view = host.hold(Viewer, { id: 1 }, context); + await flushed(); + // The view opens its own native; its dependency fails live meanwhile. + failLive(objects, dependencySelf!, new ChordError({ code: "unavailable", message: "offline" })); + await flushed(); + gate.resolve(); + await flushed(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(failures(errors)).toEqual([]); + view.release(); + }); +}); diff --git a/packages/chord/test/next/host/ctx.test.ts b/packages/chord/test/next/host/ctx.test.ts index 732ed2d3a..bf4744b76 100644 --- a/packages/chord/test/next/host/ctx.test.ts +++ b/packages/chord/test/next/host/ctx.test.ts @@ -5,7 +5,16 @@ import { withLinkedAbortSignal } from "../../../src/next/host/ctx.ts"; import type { ChordHost } from "../../../src/next/host/host.ts"; import type { Ctx, Host, Reference } from "../../../src/next/host/index.ts"; import { constant } from "../../../src/next/state/index.ts"; -import { flushed, incarnationOf, linked, loaded, rawParent, rejection, type TestPrincipal } from "../support.ts"; +import { + flushed, + incarnationOf, + linked, + loaded, + rawParent, + rejection, + Server, + type TestPrincipal, +} from "../support.ts"; const Item = local("app.item", { version: 1, params: { id: int() }, members: { id: int() } }); const Desk = local("app.desk", { @@ -61,7 +70,7 @@ describe("ctx", () => { expect(await rejection(seen.ctx!.get(Item, { id: 1 }))).toEqual({ code: "released", reason: "settled", - message: "app.item: ctx.get after its handler settled", + message: "app.item: ctx.get after its call settled", }); }); @@ -103,7 +112,7 @@ describe("ctx", () => { expect(await seen.late).toMatchObject({ code: "released", reason: "settled", - message: "app.item: ctx.get after its handler settled", + message: "app.item: ctx.get after its call settled", }); await flushed(); // The late reference's object ended with it. @@ -111,7 +120,12 @@ describe("ctx", () => { }); it("a ctx.get still waiting when its handler settles gives up: its reference is released at once", async () => { - const Far = remote("app.far", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Far = remote("app.far", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const Starter = local("app.starter", { version: 1, params: {}, members: { start: method() } }); let pending: Promise | undefined; const { host } = await loaded((h) => @@ -130,7 +144,7 @@ describe("ctx", () => { expect(await pending).toMatchObject({ code: "released", reason: "settled", - message: "app.far: ctx.get after its handler settled", + message: "app.far: ctx.get after its call settled", }); }); @@ -146,6 +160,7 @@ describe("local handlers act as the host (§4, audit)", () => { const Gate = remote("app.gate", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { who: method(undefined, str()), through: method(undefined, str()) } } }, }); const who = (ctx: Ctx) => `${ctx.principal?.kind ?? "none"} ${ctx.incarnation}`; diff --git a/packages/chord/test/next/host/dependencies.test.ts b/packages/chord/test/next/host/dependencies.test.ts new file mode 100644 index 000000000..afa5d914b --- /dev/null +++ b/packages/chord/test/next/host/dependencies.test.ts @@ -0,0 +1,385 @@ +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, keyed, local, obj, remote } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { + bindHold, + failures, + fakeTimers, + flushed, + loaded, + Notes, + rawParent, + rejection, + relayed, + Server, + Session, + statesOf, + type TestPrincipal, + transported, +} from "../support.ts"; + +const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); + +describe("the dependency check", () => { + it("breaks a dependent off a cycle with the cycle's error: it would wait on a ring that never heals", async () => { + const Ping = local("app.ping", { version: 1, params: {}, members: { n: int() } }); + const Pong = local("app.pong", { version: 1, params: {}, members: { n: int() } }); + const { host, errors } = await loaded((h) => { + h.serve(Ping, (_, self) => ({ n: self.get(Pong, {}).n })); + h.serve(Pong, (_, self) => ({ n: self.get(Ping, {}).n })); + h.serve(View, (_, self) => ({ n: self.get(Ping, {}).n })); + }); + const view = host.hold(View, { id: 1 }, context); + await flushed(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { code: "invalid", reason: "cycle" } }, + }); + expect(failures(errors)).toContain("app.view(1): broken"); + view.release(); + }); + + it("breaks a dependent whose dependency ended (gone): the request cannot come back", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + await client.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ), + }, + ], + context, + ); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const revoked = { code: "forbidden", reason: "revoked", message: "revoked" } as const; + peer.send({ kind: "gone", hold, error: { kind: "chord", error: revoked } }); + await transported(); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken", error: revoked } }); + expect(failures(errors)).toEqual(["app.view(1): broken"]); + view.release(); + }); + + it("goes live once every dependency is: one parked on two, the second binding last", async () => { + const client = createHost({ onError: () => {} }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => { + const a = self.get(Notes, { session: "S1" }).count; + const b = self.get(Notes, { session: "S2" }).count; + return { n: self.derive([a, b], ([x, y]) => x + y) }; + }), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const first = await peer.next("hold"); + const second = await peer.next("hold"); + bindHold(peer, first.hold, [{ text: "a" }]); + await transported(); + expect(view.state.value).toEqual({ kind: "loading" }); + bindHold(peer, second.hold, [{ text: "b" }, { text: "c" }]); + await transported(); + expect([view.state.value, view.n.value]).toEqual([{ kind: "live" }, 3]); + view.release(); + }); + + it("breaks a dependent whose dependency a relay's authorize refused: forbidden heals only by retry() (fix round 1, Fable)", async () => { + const errors: HostError[] = []; + const client = createHost({ onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + const forbidden = { code: "forbidden", message: "session|S1: not authorized" } as const; + peer.send({ kind: "failed", hold, cause: { kind: "broken", error: { kind: "chord", error: forbidden } } }); + await transported(); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken", error: forbidden } }); + expect(failures(errors)).toEqual(["app.view(1): broken"]); + view.release(); + }); + + it("settles a waiting ring across hosts after one round: the root cause kept (fix round 1, Opus F1)", async () => { + const A = remote("app.a", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const B = remote("app.b", { + version: 1, + params: {}, + hostAddress: () => Session({ id: "S1" }), + variants: { v: { state: { n: int() } } }, + }); + const X = local("app.x", { version: 1, params: {}, members: { n: int() } }); + const Kick = local("app.kick", { version: 1, params: {}, members: { n: int() } }); + const Front = local("app.front", { version: 1, params: {}, members: { n: int() } }); + const Back = local("app.back", { version: 1, params: {}, members: { n: int() } }); + const { workers } = await relayed({ + trusted: true, + server: (h) => { + // The server's host object starts the worker; its A and the worker's B then start in the same turns, so + // the activation chains miss the ring (M§7's gap). + h.serve(Kick, (_p, self) => ({ n: self.get(B, {}).n })); + h.serve(A, (_p, _principal, self) => ({ variant: "v", members: { n: self.get(B, {}).n } })); + }, + worker: (h) => { + h.serve(Front, (_p, self) => ({ n: self.get(A, {}).n })); + h.serve(Back, (_p, self) => ({ n: self.get(B, {}).n })); + h.serve(B, (_p, _principal, self) => { + const a = self.get(A, {}); + self.get(X, {}); + return { variant: "v", members: { n: a.n } }; + }); + }, + }); + await transported(); + const worker = workers.get("session|S1")!; + const b = worker.hold(B, {}, context); + for (let round = 0; round < 50; round++) await transported(); + const states = statesOf(await b.state.watch(context)); + for (let round = 0; round < 50; round++) await transported(); + expect(states).toEqual([]); + expect(b.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: expect.stringMatching(/: app\.x is not served here$/) } }, + }); + b.release(); + }); +}); + +describe("parked_long (dependency model §7, audit A11)", () => { + fakeTimers(); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + + it("is reported once after readyTimeout whatever the blocker: a missing local provider; nothing fails", async () => { + const { host, errors } = await loaded((h) => h.serve(View, (_, self) => ({ n: self.get(Missing, {}).n })), { + readyTimeout: 1_000, + }); + const view = host.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(999); + expect(errors).toEqual([]); + await vi.advanceTimersByTimeAsync(1); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.missing", state: "failed", since: 0 }, + ]); + await vi.advanceTimersByTimeAsync(60_000); + expect(errors).toHaveLength(1); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + view.release(); + await host.close(context); + }); + + it("names the first blocker in declaration order with its state; its clock runs from parking, with no link yet too", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ + n: self.derive([self.get(Notes, { session: "S1" }).count, self.get(Missing, {}).n], ([a, b]) => a + b), + })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(1_000); + // The remote one comes first (no link holds it yet), though the attempt shows the failed one. + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.notes(S1)", state: "loading", since: 0 }, + ]); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + view.release(); + await client.close(context); + }); + + it("keeps declaration order across registries and references: the parked cause and the report name the first (Astra)", async () => { + const Registry = defineRegistry("app.missing-registry", { version: 1 }); + const { host, errors } = await loaded( + (h) => + h.serve(View, (_, self) => { + self.registry(Registry); + return { n: self.get(Missing, {}).n }; + }), + { readyTimeout: 1_000 }, + ); + const view = host.hold(View, { id: 1 }, context); + await flushed(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { message: expect.stringMatching(/^waiting for registry app\.missing-registry: /) }, + }, + }); + await vi.advanceTimersByTimeAsync(1_000); + expect(errors).toEqual([ + { + kind: "parked_long", + object: "app.view(1)", + waitingFor: "registry app.missing-registry", + state: "failed", + since: 0, + }, + ]); + view.release(); + await host.close(context); + }); + + it("names a reference whose pinned key still loads: the episode's clock includes the load (Sol)", async () => { + const Rows = defineState("rows", { + version: 1, + keyed: true, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + }); + const Keys = local("app.keys", { version: 1, params: {}, members: { rows: keyed(obj({ n: int() })) } }); + const gate = Promise.withResolvers(); + const entered = Promise.withResolvers(); + const pinKey = ChordObject.prototype.pinKey; + const held = vi.spyOn(ChordObject.prototype, "pinKey").mockImplementation(async function ( + this: ChordObject, + member, + key, + version, + pinContext, + ) { + entered.resolve(); + await gate.promise; + return pinKey.call(this, member, key, version, pinContext); + }); + try { + const { host, errors } = await loaded( + (h) => { + h.serve(Keys, (_, self) => ({ rows: self.state(Rows) })); + h.serve(View, (_, self) => { + const row = self.get(Keys, {}).rows.at("a"); + return { n: self.derive([row], ([value]) => value.n) }; + }); + }, + { readyTimeout: 1_000 }, + ); + const view = host.hold(View, { id: 1 }, context); + await entered.promise; + await vi.advanceTimersByTimeAsync(1_000); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.keys", state: "loading", since: 0 }, + ]); + gate.resolve(); + await vi.waitFor(() => expect(view.state.value).toEqual({ kind: "live" })); + view.release(); + await host.close(context); + } finally { + gate.resolve(); + held.mockRestore(); + } + }); + + it("reports a ring across hosts whose member there waits on a failed dependency (it shows failed here)", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + // The provider's object is parked there on a failed dependency (the ring's member back to this host waits too). + const error = { + kind: "chord", + error: { code: "unavailable", message: "waiting for app.c: not served" }, + } as const; + peer.send({ kind: "failed", hold, cause: { kind: "waiting", error } }); + await vi.advanceTimersByTimeAsync(1_000); + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view(1)", waitingFor: "app.notes(S1)", state: "failed", since: 0 }, + ]); + expect(view.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + view.release(); + await client.close(context); + }); + + it("is reported once per parking episode: again for an attempt that parks after leaving parking", async () => { + const { host, errors } = await loaded((h) => h.serve(View, (_, self) => ({ n: self.get(Missing, {}).n })), { + readyTimeout: 1_000, + }); + const view = host.hold(View, { id: 1 }, context); + await vi.advanceTimersByTimeAsync(5_000); + expect(errors.map(({ kind }) => kind)).toEqual(["parked_long"]); + // A reload of its plugin replaces the parked attempt: a new episode, its clock from its own parking. + const changed = defineFacet((h) => h.serve(View, (_, self) => ({ n: self.get(Missing, {}).n }))); + await host.reload([{ name: "@test/app", facet: changed }], context); + await vi.advanceTimersByTimeAsync(999); + expect(errors).toHaveLength(1); + await vi.advanceTimersByTimeAsync(1); + expect(errors[1]).toEqual({ + kind: "parked_long", + object: "app.view(1)", + waitingFor: "app.missing", + state: "failed", + since: 5_000, + }); + view.release(); + await host.close(context); + }); + + it("is not reported for an attempt that leaves parking within readyTimeout: its timer stops", async () => { + const errors: HostError[] = []; + const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); + const peer = await rawParent(client); + const facet = defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await client.load([{ name: "@test/view", facet }], context); + const view = client.hold(View, { id: 1 }, context); + const { hold } = await peer.next("hold"); + await vi.advanceTimersByTimeAsync(500); + bindHold(peer, hold, [{ text: "a" }]); + await vi.advanceTimersByTimeAsync(5_000); + expect(view.state.value).toEqual({ kind: "live" }); + expect(errors).toEqual([]); + view.release(); + await client.close(context); + }); +}); + +describe("a getter of a parked object (fix round 1, Astra 6)", () => { + it("rejects with the last parked cause after its dependency fell back to loading, the newest of several", async () => { + const host = createHost({ onError: () => {} }); + const peer = await rawParent(host); + await host.load( + [ + { + name: "@test/view", + facet: defineFacet((h) => + h.serve(View, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ), + }, + ], + context, + ); + const pending = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 80 })); + const { hold } = await peer.next("hold"); + for (const message of ["first absence", "specific provider absence"]) { + const error = { kind: "chord", error: { code: "unavailable", message } } as const; + peer.send({ kind: "failed", hold, cause: { kind: "waiting", error } }); + await transported(); + } + peer.send({ kind: "loading", hold }); + await transported(); + expect(await pending).toMatchObject({ + code: "unavailable", + message: "waiting for app.notes(S1): specific provider absence", + }); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/facet.test.ts b/packages/chord/test/next/host/facet.test.ts index 54fe1113d..619cea025 100644 --- a/packages/chord/test/next/host/facet.test.ts +++ b/packages/chord/test/next/host/facet.test.ts @@ -1,6 +1,8 @@ import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { int, local } from "../../../src/next/contracts/index.ts"; -import { defineFacet, type Facet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; +import { defineFacet, type Facet, type FacetHost, type FacetSelf, type Host } from "../../../src/next/host/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import { loaded } from "../support.ts"; @@ -17,13 +19,37 @@ describe("facets", () => { ); }); - it("only declare: a setup takes the host alone, which has no documents, routines or natives", () => { - expectTypeOf["setup"]>>().toEqualTypeOf<[FacetHost<{ kind: "owner" }>]>(); + it("only declare: a setup takes the host and a self that only contributes; no documents, routines or natives", () => { + expectTypeOf["setup"]>>().toEqualTypeOf< + [FacetHost<{ kind: "owner" }>, FacetSelf] + >(); + expectTypeOf().toEqualTypeOf<"contribute">(); expectTypeOf().not.toHaveProperty("state"); expectTypeOf().not.toHaveProperty("on"); expectTypeOf().not.toHaveProperty("onLive"); }); + it("contribute at most one value per registry, only while their setup runs", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const twice = await loaded((_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + }); + expect(twice.failed.map(({ error }) => (error as Error).message)).toEqual([ + 'plugin "@test/app": @test/app: registry app.names is contributed to twice', + ]); + await twice.host.close(context); + let later: FacetSelf | undefined; + const { host } = await loaded((host, self) => { + host.serve(Names, { compose: (table) => table.length }); + later = self; + }); + expect(() => later!.contribute(Names, "late")).toThrow( + "@test/app: self.contribute is valid only in the facet's setup", + ); + await host.close(context); + }); + it("are typed by exactly the app's principal type", () => { type App = { kind: "owner" } | { kind: "guest"; session: string }; const facet = defineFacet(() => {}); diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index 64a291b43..0ae2d5361 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; -import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; -import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { evolve, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { broken } from "../../../src/next/data/index.ts"; import { type Ctx, @@ -13,6 +13,7 @@ import { import type { ChordHost } from "../../../src/next/host/internal.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; import { jsonPart } from "../../../src/next/link/internal.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import { sqliteBackend } from "../../../src/next/store/backend.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; @@ -31,8 +32,12 @@ import { partValue, RawPeer, rejection, + relayed, + SERVER_ADDRESS, + Server, Session, serveNotes, + statesOf, type TestPrincipal, testCodec, transported, @@ -74,22 +79,46 @@ describe("load", () => { await host.close(context); }); - it("refuses invalid and duplicate plugin names, double serves and a second load", async () => { + it("leaves out and reports invalid and duplicate plugin names and double serves (the later); refuses a second load", async () => { const empty = defineFacet(() => {}); const info = defineFacet((host) => host.serve(Info, () => ({ title: constant("t") }))); - await expect(createHost(noErrors).load([{ name: "Bad Name", facet: empty }], context)).rejects.toThrow( - '"Bad Name" is not a valid plugin name', - ); + const loaded = async (facets: { readonly name: string; readonly facet: typeof empty }[]) => { + const host = createHost(noErrors); + const { failed } = await host.load(facets, context); + await host.close(context); + return failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + }; + expect(await loaded([{ name: "Bad Name", facet: empty }])).toEqual([ + ["Bad Name", '"Bad Name" is not a valid plugin name'], + ]); const twice = [ { name: "a", facet: empty }, { name: "a", facet: empty }, ]; - await expect(createHost(noErrors).load(twice, context)).rejects.toThrow('plugin "a" is loaded twice'); + expect(await loaded(twice)).toEqual([["a", 'plugin "a" is loaded twice']]); + const throwing = defineFacet(() => { + throw new Error("first failed"); + }); + // The first one's setup failing does not make the second one's name free (Sol). + const failing = createHost(noErrors); + const report = await failing.load( + [ + { name: "a", facet: throwing }, + { name: "a", facet: info }, + ], + context, + ); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + ["a", 'plugin "a": first failed'], + ["a", 'plugin "a" is loaded twice'], + ]); + expect(failing.hold(Info, {}, context).state.value).toMatchObject({ kind: "failed" }); + await failing.close(context); const served = [ { name: "a", facet: info }, { name: "b", facet: info }, ]; - await expect(createHost(noErrors).load(served, context)).rejects.toThrow("app.info is served twice"); + expect(await loaded(served)).toEqual([["b", 'plugin "b": app.info is served twice (plugin "a")']]); const host = createHost(noErrors); await host.load([], context); await expect(host.load([], context)).rejects.toThrow("load runs once per host"); @@ -151,19 +180,296 @@ describe("the start barrier", () => { await vi.advanceTimersByTimeAsync(1); expect(done).toBe(true); }); + + it("waits for a local blocker's first outcome: a host object parked on another one here (CodeMode → Harness)", async () => { + const Harness = local("app.harness", { version: 1, params: {}, members: { n: int() } }); + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const opened = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(Harness, (_, self) => { + self.open( + () => opened.promise, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(CodeMode, (_, self) => ({ n: self.get(Harness, {}).n })); + }); + let done = false; + const loading = host.load([{ name: "@app/core", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(done).toBe(false); + opened.resolve(); + await loading; + expect(host.hold(CodeMode, {}, context).state.value).toEqual({ kind: "live" }); + }); + + it("ends its pass at a local blocker's own failure: the dependent parked failed { waiting } has its first outcome", async () => { + const Harness = local("app.harness", { version: 1, params: {}, members: { n: int() } }); + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const { host, errors } = await loaded((h) => { + h.serve(Harness, (_, self) => { + self.open( + async () => { + throw new Error("offline"); + }, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(CodeMode, (_, self) => ({ n: self.get(Harness, {}).n })); + }); + expect(failures(errors)).toEqual(["app.harness: transient"]); + expect(host.hold(CodeMode, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.harness: app.harness: offline" } }, + }); + }); + + it("ends its pass at a remote blocker: a wait on another host is a first outcome", async () => { + const CodeMode = local("app.codemode", { version: 1, params: {}, members: { n: int() } }); + const host = createHost(noErrors); + const [a, b] = memoryLinkPair(); + host.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(b); + const facet = defineFacet((h) => + h.serve(CodeMode, (_, self) => ({ n: self.get(Notes, { session: "S1" }).count })), + ); + await host.load([{ name: "@app/codemode", facet }], context); + expect(await parent.next("hold")).toMatchObject({ type: Notes.name }); + expect(host.hold(CodeMode, {}, context).state.value).toEqual({ kind: "loading" }); + }); + + it("is settled by host.close: an object of its cohort that ends before its first outcome settles it as ended", async () => { + const host = createHost(noErrors); + const facet = defineFacet((h) => + h.serve(Info, (_, self) => { + self.open( + () => new Promise(() => {}), + () => {}, + ); + return { title: constant("t") }; + }), + ); + let done = false; + const loading = host.load([{ name: "a", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(done).toBe(false); + await host.close(context); + await loading; + expect(done).toBe(true); + }); + + it("waits for a local blocker parked failed { waiting } on one still in its first attempt (fix round 1, Sol 4)", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + const Middle = local("app.middle", { version: 1, params: { id: int() }, members: { n: int() } }); + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + const gate = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(Slow, (_, self) => { + self.open( + () => gate.promise, + () => {}, + ); + return { n: constant(1) }; + }); + h.serve(Middle, ({ id }, self) => { + const slow = self.get(Slow, { id }); + self.get(Missing, {}); + return { n: slow.n }; + }); + h.serve(Core, (_, self) => ({ n: self.get(Middle, { id: 1 }).n })); + }); + let done = false; + const loading = host.load([{ name: "@app/core", facet }], context).then(() => { + done = true; + }); + await flushed(); + expect(host.hold(Middle, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + expect(done).toBe(false); + gate.resolve(); + await loading; + await host.close(context); + }); + + it("waits for it whichever it declares first (fix round 1, Astra 3)", async () => { + const A = local("app.a", { version: 1, params: {}, members: {} }); + const B = local("app.b", { version: 1, params: { id: int() }, members: {} }); + const Slow = local("app.slow", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + const gate = Promise.withResolvers(); + const host = createHost(noErrors); + const facet = defineFacet((h) => { + h.serve(A, (_, self) => { + self.get(B, { id: 1 }); + return {}; + }); + h.serve(B, (_, self) => { + self.get(Missing, {}); + self.get(Slow, { id: 1 }); + return {}; + }); + h.serve(Slow, (_, self) => { + self.open( + () => gate.promise, + () => {}, + ); + return {}; + }); + }); + let done = false; + const loading = host.load([{ name: "@app/a", facet }], context).then(() => { + done = true; + }); + await flushed(); + const premature = done; + gate.resolve(); + await loading; + await host.close(context); + expect(premature).toBe(false); + }); +}); + +describe("startup across hosts (dependency model §6)", () => { + // Timers never advance: nothing here may wait for a deadline. + fakeTimers(); + + /** On the server, waiting for a worker's `Agent`. */ + const Bootstrap = local("app.bootstrap", { version: 1, params: {}, members: { n: int() } }); + /** On worker S1, waiting for the server's `Catalog`. */ + const Agent = remote("app.agent", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, + }); + const Catalog = remote("app.catalog", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + + it("starts two hosts that would have deadlocked (Sol's probe): an acyclic graph needs no timeout", async () => { + const { server, errors } = await relayed({ + server: (h) => { + h.serve(Bootstrap, (_, self) => ({ n: self.get(Agent, { session: "S1" }).n })); + h.serve(Catalog, () => ({ variant: "v", members: { n: constant(7) } })); + }, + worker: (h) => { + // The worker's own host object waits for the server's catalog during the worker's load. + h.serve(Agent, (_params, _principal, self) => ({ variant: "v", members: { n: self.get(Catalog, {}).n } })); + }, + }); + await transported(); + await transported(); + const bootstrap = server.hold(Bootstrap, {}, context); + expect(bootstrap.state.value).toEqual({ kind: "live" }); + expect(bootstrap.n.value).toBe(7); + expect([errors.server, errors.worker]).toEqual([[], []]); + }); + + it("places a hold arriving before the publication here, not over the parent", async () => { + const worker = createHost({ + address: Session({ id: "S1" }), + principal: { kind: "worker", session: "S1" }, + ...noErrors, + }); + const [a, b] = memoryLinkPair(); + worker.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); + const parent = new RawPeer(b); + const [c, d] = memoryLinkPair(); + worker.accept(c, { principal: { kind: "owner" }, codec: testCodec() }, context); + const child = new RawPeer(d); + child.send({ + kind: "hold", + hold: 0, + type: PlacedNotes.name, + version: PlacedNotes.version, + fingerprint: PlacedNotes.fingerprint, + params: jsonPart({ session: "S1" }), + address: { kind: "session", params: { id: "S1" } }, + }); + const mine = worker.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(mine.state.value).toEqual({ kind: "loading" }); + await worker.load( + [{ name: "@test/w", facet: defineFacet((h) => serveNotes(h, PlacedNotes)) }], + context, + ); + expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + await transported(); + expect(mine.state.value).toEqual({ kind: "live" }); + expect(parent.kinds()).toEqual(["hello"]); + }); + + it("wakes a parked chain over three hosts in order, its causes passed on (tab → worker → server)", async () => { + const Chat = remote("app.chat", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { n: int() } } }, + }); + const order: string[] = []; + let catalogs = 0; + const { tab } = await relayed({ + server: (h) => + h.serve(Catalog, (_params, _principal, self) => { + const attempt = ++catalogs; + self.open( + async () => { + if (attempt === 1) throw new Error("offline"); + }, + () => {}, + ); + self.onLive(async () => void order.push("catalog")); + return { variant: "v", members: { n: constant(attempt) } }; + }), + worker: (h) => + h.serve(Chat, (_params, _principal, self) => { + const catalog = self.get(Catalog, {}); + self.onLive(async () => void order.push("chat")); + return { variant: "v", members: { n: catalog.n } }; + }), + }); + const view = tab.hold(Chat, { session: "S1" }, context); + for (let round = 0; round < 5; round++) await transported(); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.catalog: app.catalog: offline" } }, + retryAt: undefined, + }); + // The catalog's own backoff (1 s) is the only timer: it retries, and the chain goes on in order. + await vi.advanceTimersByTimeAsync(1_000); + for (let round = 0; round < 5; round++) await transported(); + expect(view.state.value).toEqual({ kind: "live" }); + expect(order).toEqual(["catalog", "chat"]); + expect(view.n.value).toBe(2); + }); }); describe("get", () => { - it("refuses an unserved type and a contract mismatch", async () => { + it("waits for an unserved type and a contract mismatch until its readyTimeout, then rejects with the cause", async () => { const { host } = await loaded((h) => h.serve(Info, () => ({ title: constant("t") }))); - expect(await rejection(host.get(Page, { id: 1 }, context))).toEqual({ + const quick = { readyTimeout: 5 }; + expect(await rejection(host.get(Page, { id: 1 }, context, quick))).toEqual({ code: "unavailable", message: "app.page is not served here", }); const InfoV2 = local("app.info", { version: 2, params: {}, members: { title: str() } }); const Changed = local("app.info", { version: 1, params: {}, members: { title: int() } }); - expect(await rejection(host.get(InfoV2, {}, context))).toMatchObject({ code: "contract_mismatch" }); - expect(await rejection(host.get(Changed, {}, context))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.get(InfoV2, {}, context, quick))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.get(Changed, {}, context, quick))).toMatchObject({ code: "contract_mismatch" }); const Same = local("app.info", { version: 1, params: {}, members: { title: str() } }); expect((await host.get(Same, {}, context)).title.value).toBe("t"); }); @@ -482,13 +788,24 @@ describe("getters", () => { forEachCodec(() => { describe("principals and where a reference goes", () => { - const Who = remote("app.who", { version: 1, params: { id: int() }, variants: { v: { state: { who: str() } } } }); - const Outer = remote("app.outer", { version: 1, params: {}, variants: { v: { state: { who: str() } } } }); + const Who = remote("app.who", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { who: str() } } }, + }); + const Outer = remote("app.outer", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { who: str() } } }, + }); const Inner = local("app.inner", { version: 1, params: { id: int() }, members: { who: str() } }); const Desk = local("app.desk", { version: 1, params: {}, members: { who: method(undefined, str()) } }); const Gate = remote("app.gate", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { who: method(undefined, str()) } } }, }); type P = TestPrincipal; @@ -514,8 +831,36 @@ forEachCodec(() => { const { server } = await linked((h) => serveAll(h, factories)); expect((await server.get(Who, { id: 1 }, context)).who.value).toBe("server"); const client = createHost

({ onError: () => {} }); - const loading = client.load([{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], context); - await expect(loading).rejects.toThrow("app.who: a host without a principal serves only local types"); + const report = await client.load( + [{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], + context, + ); + expect(report.failed).toEqual([ + { + plugin: "@test/app", + error: expect.objectContaining({ + message: 'plugin "@test/app": app.who: a host without a principal serves only local types', + }), + }, + ]); + await client.close(context); + }); + + it("a host without an address serves no remote type, whatever its principal", async () => { + const host = createHost

({ principal: { kind: "server" }, onError: () => {} }); + const report = await host.load( + [{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], + context, + ); + expect(report.failed).toEqual([ + { + plugin: "@test/app", + error: expect.objectContaining({ + message: 'plugin "@test/app": app.who: a host without an address serves only local types', + }), + }, + ]); + await host.close(context); }); it("ctx.get acts as ctx.principal: a guest's call gets the guest's view; a local handler's acts as the host", async () => { @@ -533,6 +878,7 @@ forEachCodec(() => { it("identity includes the principal key: equal keys share one object", async () => { const factories: string[] = []; const host = createHost

({ + address: Server(), principal: { kind: "guest", session: "S1" }, principalKey: (principal) => principal.kind, onError: () => {}, @@ -549,6 +895,7 @@ forEachCodec(() => { version: Who.version, fingerprint: Who.fingerprint, params: jsonPart({ id: 1 }), + address: SERVER_ADDRESS, }); await peer.next("bound"); expect(factories).toEqual(["1 guest"]); @@ -562,6 +909,7 @@ forEachCodec(() => { version: Who.version, fingerprint: Who.fingerprint, params: jsonPart({ id: 1 }), + address: SERVER_ADDRESS, }); await owner.next("bound"); expect(factories).toEqual(["1 guest", "1 owner"]); @@ -569,7 +917,12 @@ forEachCodec(() => { it("remote objects with empty params are not activated by load; they live while referenced", async () => { const factories: string[] = []; - const Empty = remote("app.empty", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Empty = remote("app.empty", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { server } = await linked((h) => h.serve(Empty, (_, principal) => { factories.push(principal.kind); @@ -585,7 +938,11 @@ forEachCodec(() => { it("a local type never crosses a link; a remote type without a parent link waits for one", async () => { const { client } = await linked(() => {}); - expect(await rejection(client.get(Inner, { id: 1 }, context))).toEqual({ + expect(client.hold(Inner, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, + }); + expect(await rejection(client.get(Inner, { id: 1 }, context, { readyTimeout: 5 }))).toEqual({ code: "unavailable", message: "app.inner is not served here", }); @@ -618,8 +975,83 @@ forEachCodec(() => { }); }); -describe("a load whose setup throws", () => { - it("closes the host as close does, links included, and rejects with the setup's error", async () => { +describe("host.registry and the host's close (fix round 1)", () => { + const Absent = defineRegistry("app.absent", { version: 1 }); + + it("ends a waiting host.registry at the close with released/closed, its deadline timer with it (Sol 5, Astra 5)", async () => { + const host = createHost(noErrors); + await host.load([], context); + let outcome: unknown; + void rejection(host.registry(Absent, context, { readyTimeout: 60_000 })).then((error) => { + outcome = error; + }); + await flushed(); + await host.close(context); + await flushed(); + expect(outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("ends one still waiting for the publication at the close", async () => { + const host = createHost(noErrors); + const waiting = rejection(host.registry(Absent, context, { readyTimeout: 60_000 })); + await host.close(context); + expect(await waiting).toMatchObject({ code: "released", reason: "closed" }); + }); +}); + +describe("the selected provider's own checks for a request made here (fix round 1)", () => { + const Here = hostKind("app-here"); + const Elsewhere = hostKind("app-elsewhere"); + const shape = { version: 1, params: {}, variants: { view: { state: { n: int() } } } } as const; + + it("waits a same-host remote request the provider's code places elsewhere: invalid/host_address (Astra 4)", async () => { + const Consumer = remote("app.placed", { ...shape, hostAddress: () => Here() }); + const Provider = remote("app.placed", { ...shape, hostAddress: () => Elsewhere() }); + const host = createHost({ address: Here(), principal: { kind: "owner" }, onError: () => {} }); + const facet = defineFacet((h) => h.serve(Provider, () => ({ variant: "view", members: { n: constant(1) } }))); + await host.load([{ name: "@app/provider", facet }], context); + const ref = host.hold(Consumer, {}, context); + await flushed(); + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "invalid", reason: "host_address" } }, + }); + ref.release(); + await host.close(context); + }); + + it("fails broken when the provider's hostAddress throws: its own code (Sol 7)", async () => { + const Consumer = remote("app.remote", { ...shape, hostAddress: () => Server() }); + const Provider = remote("app.remote", { + ...shape, + hostAddress: () => { + throw new Error("provider address bug"); + }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const client = createHost({ onError: () => {} }); + const facet = defineFacet((h) => h.serve(Provider, () => ({ variant: "view", members: { n: constant(1) } }))); + await server.load([{ name: "@app/provider", facet }], context); + await client.load([], context); + const [down, up] = memoryLinkPair(); + server.accept(down, { principal: { kind: "server" } }, context); + client.connect(up, { principal: { kind: "server" } }, context); + const held = client.hold(Consumer, {}, context); + await transported(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "broken", error: { message: "app.remote: provider address bug" } }, + }); + // The same throw for a request made on the provider's host fails it broken too. + const here = server.hold(Consumer, {}, context); + expect(here.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await client.close(context); + await server.close(context); + }); +}); + +describe('a load whose setup throws (decision "Per plugin at startup and reload")', () => { + it("leaves that plugin out and reports it; the others publish, the host and its links stay open", async () => { const host = createHost({ principal: { kind: "server" }, onError: () => {} }); const [a, b] = memoryLinkPair(); const [c, d] = memoryLinkPair(); @@ -627,17 +1059,31 @@ describe("a load whose setup throws", () => { host.connect(c, { principal: { kind: "server" }, codec: testCodec() }, context); const child = new RawPeer(b); const parent = new RawPeer(d); - const facet = defineFacet(() => { + const broken = defineFacet(() => { throw new Error("setup threw"); }); - await expect(host.load([{ name: "@test/app", facet }], context)).rejects.toThrow("setup threw"); + const report = await host.load( + [ + { name: "@test/broken", facet: broken }, + { + name: "@test/app", + facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))), + }, + ], + context, + ); + expect(report.failed).toEqual([ + { plugin: "@test/broken", error: expect.objectContaining({ message: 'plugin "@test/broken": setup threw' }) }, + ]); await transported(); - expect([child.closed, parent.closed]).toEqual([true, true]); - expect(await rejection(host.get(Page, { id: 1 }, context))).toEqual({ - code: "released", - reason: "closed", - message: "the host is closed", + expect([child.closed, parent.closed]).toEqual([false, false]); + expect((await host.get(Info, {}, context)).title.value).toBe("t"); + // What the left-out plugin would have served waits as any missing type does. + expect(host.hold(Page, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, }); + await host.close(context); }); }); @@ -660,23 +1106,29 @@ describe("acquiring: hold and get", () => { const offline = { code: "unavailable", message: "app.svc(1): offline" }; - it("invalid params and a local type not served here: at once", async () => { - const host = createHost({ ...noErrors, principal: { kind: "server" } }); + it("invalid params at once; a local type not served here: hold waits failed, get rejects at its readyTimeout", async () => { + const host = createHost({ address: Server(), ...noErrors, principal: { kind: "server" } }); await host.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const params = { code: "invalid", reason: "params" }; expect(() => host.hold(Notes, { session: 1 } as never, context)).toThrow("params"); expect(await rejection(host.get(Notes, { session: 1 } as never, context))).toMatchObject(params); const unserved = { code: "unavailable", message: "app.info is not served here" }; - expect(() => host.hold(Info, {}, context)).toThrow(unserved.message); - expect(await rejection(host.get(Info, {}, context))).toEqual(unserved); + expect(host.hold(Info, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: unserved }, + retryAt: undefined, + }); + const got = rejection(host.get(Info, {}, context, { readyTimeout: 5 })); + await vi.advanceTimersByTimeAsync(5); + expect(await got).toEqual(unserved); }); - it("before load resolved: hold is loading, get waits; once it resolved both bind, or an unserved type fails", async () => { + it("before load resolved: hold is loading, get waits; once it resolved both bind, or an unserved type waits", async () => { const host = createHost(noErrors); const held = host.hold(Info, {}, context); const unserved = host.hold(Page, { id: 1 }, context); const got = host.get(Info, {}, context); - const refused = rejection(host.get(Page, { id: 1 }, context)); + const refused = rejection(host.get(Page, { id: 1 }, context, { readyTimeout: 50 })); expect([held.state.value, unserved.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); await host.load( [{ name: "@test/app", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], @@ -688,9 +1140,10 @@ describe("acquiring: hold and get", () => { const error = { code: "unavailable", message: "app.page is not served here" }; expect(unserved.state.value).toMatchObject({ kind: "failed", - cause: { kind: "broken", error }, + cause: { kind: "waiting", error }, retryAt: undefined, }); + await vi.advanceTimersByTimeAsync(50); expect(await refused).toEqual(error); }); @@ -814,8 +1267,8 @@ forEachCodec(() => { return { host, child: new RawPeer(b), parent, errors }; } - /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming `address` when given. */ - function holdOf(peer: RawPeer, session: string, address?: string, type: string = PlacedNotes.name, hold = 0) { + /** Sends a hold of `PlacedNotes` (or `type`) for `session`, naming the session `address`. */ + function holdOf(peer: RawPeer, session: string, address: string, type: string = PlacedNotes.name, hold = 0) { peer.send({ kind: "hold", hold, @@ -823,13 +1276,13 @@ forEachCodec(() => { version: PlacedNotes.version, fingerprint: PlacedNotes.fingerprint, params: jsonPart({ session }), - ...(address === undefined ? {} : { address: { kind: "session", params: { id: address } } }), + address: { kind: "session", params: { id: address } }, }); } const misplaced = { code: "invalid", - reason: "placement", + reason: "host_address", message: "app.placed-notes: placed on session, this host is session|S1", }; @@ -839,28 +1292,28 @@ forEachCodec(() => { expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); }); - it("refuses a hold served here whose params place it elsewhere, whatever address it names: invalid/placement (race 5)", async () => { + it("keeps a hold naming this host whose params the code here places elsewhere waiting: invalid/host_address (code skew, M§1)", async () => { const { child } = await worker(); holdOf(child, "S2", "S1"); expect(await child.next("failed")).toMatchObject({ hold: 0, - cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, - }); - // A refused hold is kept (a retry starts it again): the next hold takes the next id. - holdOf(child, "S2", undefined, PlacedNotes.name, 1); - expect(await child.next("failed")).toMatchObject({ - hold: 1, - cause: { kind: "broken", error: { kind: "chord", error: misplaced } }, + cause: { kind: "waiting", error: { kind: "chord", error: misplaced } }, }); + // A refused hold is kept (a publication or a retry starts it again): the next hold takes the next id. + holdOf(child, "S2", "S1", PlacedNotes.name, 1); + expect(await child.next("failed")).toMatchObject({ hold: 1, cause: { kind: "waiting" } }); }); - it("places a hold served here by its params, not by the address it names (that only routes it)", async () => { - const { child } = await worker(); - holdOf(child, "S1", "S2"); - expect((await child.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + it("places a hold by the address it names, never by the name it serves (a relay never captures it)", async () => { + const { child, parent } = await worker({ parent: true }); + holdOf(child, "S2", "S2"); + const relayed = await parent!.next("hold"); + expect(relayed).toMatchObject({ type: PlacedNotes.name, address: { kind: "session", params: { id: "S2" } } }); + holdOf(child, "S1", "S2", PlacedNotes.name, 1); + expect(await parent!.next("hold")).toMatchObject({ hold: 1 }); }); - it("sends a reference made here whose placement is another host over the parent link, though it serves the type", async () => { + it("sends a reference made here whose host address is another host over the parent link, though it serves the type", async () => { const { host, parent } = await worker({ parent: true }); const elsewhere = host.hold(PlacedNotes, { session: "S2" }, context); const hold = await parent!.next("hold"); @@ -871,7 +1324,7 @@ forEachCodec(() => { it("sends a hold it cannot serve or route over the parent link, unparsed", async () => { const { child, parent } = await worker({ parent: true }); - holdOf(child, "S9", undefined, "app.elsewhere"); + holdOf(child, "S9", "S9", "app.elsewhere"); const relayed = await parent!.next("hold"); expect(relayed).toMatchObject({ type: "app.elsewhere" }); expect(partValue(relayed.params)).toEqual({ session: "S9" }); @@ -879,7 +1332,7 @@ forEachCodec(() => { it("never sends a hold from the parent back to it: unavailable (race 8)", async () => { const { parent } = await worker({ parent: true }); - holdOf(parent!, "S9", undefined, "app.elsewhere"); + holdOf(parent!, "S9", "S9", "app.elsewhere"); expect(await parent!.next("failed")).toMatchObject({ hold: 0, cause: { @@ -892,25 +1345,25 @@ forEachCodec(() => { it("refuses a hold it cannot serve or route without a parent link: unavailable", async () => { const { child } = await worker(); - holdOf(child, "S9", undefined, "app.elsewhere"); + holdOf(child, "S9", "S9", "app.elsewhere"); expect(await child.next("failed")).toMatchObject({ cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, }); }); - it("fails a reference broken with what its placement throws", async () => { + it("fails a reference broken with what its hostAddress throws", async () => { const Throwing = remote("app.throwing", { version: 1, params: {}, - placement: () => { - throw new Error("placement threw"); + hostAddress: () => { + throw new Error("hostAddress threw"); }, variants: { v: { state: { n: int() } } }, }); const client = createHost(noErrors); await client.load([], context); expect(await rejection(client.get(Throwing, {}, context))).toMatchObject({ - message: "app.throwing: placement threw", + message: "app.throwing: hostAddress threw", }); expect(client.hold(Throwing, {}, context).state.value).toMatchObject({ kind: "failed" }); }); @@ -950,3 +1403,317 @@ describe("the host's store", () => { await store.close(context); }); }); + +/** A plugin's facet serving `Info` with the title `title` (a new facet value each call: a changed plugin). */ +function infoFacet(title: string) { + return defineFacet((h) => h.serve(Info, () => ({ title: constant(title) }))); +} + +describe("reload: the API (slice 7b §1)", () => { + it("throws before load resolved and rejects released/closed after close", async () => { + const host = createHost(noErrors); + expect(() => host.reload([], context)).toThrow("reload runs after load"); + await host.load([], context); + await host.close(context); + expect(await rejection(host.reload([], context))).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("reports plugins added, changed and removed; the same facet object is unchanged and untouched", async () => { + const core = infoFacet("core"); + const host = createHost(noErrors); + const kept = defineFacet((h) => h.serve(Counter, () => ({ count: constant(1), bump: async () => null }))); + await host.load( + [ + { name: "@app/core", facet: kept }, + { name: "@app/info", facet: core }, + ], + context, + ); + const counter = await host.get(Counter, {}, context); + const report = await host.reload( + [ + { name: "@app/core", facet: kept }, + { name: "@app/page", facet: defineFacet((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))) }, + ], + context, + ); + expect(report.plugins).toEqual({ added: ["@app/page"], changed: [], removed: ["@app/info"] }); + expect(report.objects).toEqual([{ object: "app.info", kind: "ended" }]); + // Untouched: the same reference stays live on the same object. + expect(counter.state.value).toEqual({ kind: "live" }); + expect((await host.get(Page, { id: 3 }, context)).id.value).toBe(3); + await host.close(context); + }); + + it("runs reloads one after another", async () => { + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const order: string[] = []; + const first = host.reload([{ name: "@app/info", facet: infoFacet("b") }], context).then(() => order.push("b")); + const second = host.reload([{ name: "@app/info", facet: infoFacet("c") }], context).then(() => order.push("c")); + await Promise.all([first, second]); + expect(order).toEqual(["b", "c"]); + expect((await host.get(Info, {}, context)).title.value).toBe("c"); + await host.close(context); + }); + + it("an abort before the commit rejects aborted and changes nothing", async () => { + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const { context: aborted, cancel } = withCancel(context); + cancel(new Error("stop")); + expect(await rejection(host.reload([{ name: "@app/info", facet: infoFacet("b") }], aborted))).toMatchObject({ + code: "aborted", + }); + expect((await host.get(Info, {}, context)).title.value).toBe("a"); + await host.close(context); + }); + + it("an abort after the commit rejects aborted naming what the commit applied, which stands", async () => { + const host = createHost(noErrors); + await host.load( + [ + { name: "@app/info", facet: infoFacet("a") }, + { name: "@app/gone", facet: defineFacet(() => {}) }, + ], + context, + ); + const info = await host.get(Info, {}, context); + const { context: aborting, cancel } = withCancel(context); + // The abort lands right after the commit (its publication batch), while the reload waits for its report. + const { batch } = host as ChordHost; + const commit = batch.commit.bind(batch); + const hook = vi.spyOn(batch, "commit").mockImplementation((adopt, ctx) => { + commit(adopt, ctx); + if (ctx === aborting) cancel(new Error("stop")); + }); + const reloading = host.reload( + [ + { name: "@app/info", facet: infoFacet("b") }, + { name: "@app/new", facet: defineFacet(() => {}) }, + ], + aborting, + ); + const error = await rejection(reloading); + hook.mockRestore(); + expect(error).toMatchObject({ + code: "aborted", + details: { committed: { added: ["@app/new"], changed: ["@app/info"], removed: ["@app/gone"], failed: [] } }, + }); + expect(info.title.value).toBe("b"); + await host.close(context); + }); + + it("reports a setup's or a factory's throw as failed, naming the plugin, which stays on its old code", async () => { + const host = createHost(noErrors); + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const info = await host.get(Info, {}, context); + const throwing = defineFacet(() => { + throw new Error("setup threw"); + }); + const failedOf = async (facets: { readonly name: string; readonly facet: typeof throwing }[]) => { + const report = await host.reload(facets, context); + return report.failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + }; + expect(await failedOf([{ name: "@app/info", facet: throwing }])).toEqual([ + ["@app/info", 'plugin "@app/info": setup threw'], + ]); + const factoryThrows = defineFacet((h) => + h.serve(Info, () => { + throw new Error("factory threw"); + }), + ); + expect(await failedOf([{ name: "@app/info", facet: factoryThrows }])).toEqual([ + ["@app/info", 'plugin "@app/info", app.info: factory threw'], + ]); + // A type another plugin serves already fails the plugin with new code; an added one is left out. + const twice = [ + { name: "@app/info", facet: infoFacet("b") }, + { name: "@app/other", facet: infoFacet("c") }, + ]; + const report = await host.reload(twice, context); + expect(report.plugins).toEqual({ added: [], changed: ["@app/info"], removed: [] }); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + ["@app/other", 'plugin "@app/other": app.info is served twice (plugin "@app/info")'], + ]); + expect(info.title.value).toBe("b"); + expect(info.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("reload: the commit (slice 7b §3)", () => { + it("serves added types, drops removed ones and their presence lists, and wakes requests per name", async () => { + const host = createHost(noErrors) as unknown as ChordHost; + await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); + const page = host.hold(Page, { id: 1 }, context); + const other = host.hold(Core, {}, context); + await flushed(); + expect(page.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + const presence = host.references(Info); + const otherStates = statesOf(await other.state.watch(context)); + await host.reload( + [ + { name: "@app/info", facet: infoFacet("a") }, + { name: "@app/page", facet: defineFacet((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))) }, + ], + context, + ); + await flushed(); + expect(page.state.value).toEqual({ kind: "live" }); + expect(page.id.value).toBe(1); + // Not a name the reload serves: still waiting, not checked again. + expect(other.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(otherStates).toEqual([]); + await host.reload( + [{ name: "@app/page", facet: defineFacet((h) => h.serve(Page, ({ id }) => ({ id: constant(id) }))) }], + context, + ); + expect(host.references(Info)).not.toBe(presence); + await host.close(context); + }); + + it("replaces a reloaded registry's provider and its plugin's contributions; a removed provider's readers wait", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const provider = (separator: string) => + defineFacet((h, self) => { + h.serve(Names, { compose: (table) => table.map(({ value }) => value).join(separator) }); + self.contribute(Names, "own"); + }); + const contributor = (name: string) => defineFacet((_h, self) => self.contribute(Names, name)); + const host = createHost(noErrors); + await host.load( + [ + { name: "@app/names", facet: provider(",") }, + { name: "@app/x", facet: contributor("x") }, + ], + context, + ); + const names = await host.registry(Names, context); + expect(names.value).toBe("own,x"); + await host.reload( + [ + { name: "@app/names", facet: provider("+") }, + { name: "@app/x", facet: contributor("y") }, + ], + context, + ); + expect(names.value).toBe("own+y"); + await host.reload([{ name: "@app/x", facet: contributor("z") }], context); + expect(() => names.value).toThrow("no plugin serves it here"); + await host.reload( + [ + { name: "@app/names", facet: provider(";") }, + { name: "@app/x", facet: contributor("z") }, + ], + context, + ); + expect(names.value).toBe("own;z"); + await host.close(context); + }); +}); + +/** A worker's title per session (v1: a string). */ +const Title = remote("app.title", { + version: 1, + params: { session: str() }, + hostAddress: ({ session }) => Session({ id: session }), + variants: { v: { state: { title: str() } } }, +}); + +/** `Title` v2: an object; v1 consumers map back through `accepts`. */ +const TitleV2 = evolve(Title, (e) => e.state("v", "title", obj({ text: str() }))); + +describe("reload: registries' older readers (slice 7b §3)", () => { + it("a reader of an older version follows the new provider's accepts[old].value; one it no longer serves waits", async () => { + const NamesV1 = defineRegistry("app.names", { version: 1 }); + const NamesV2 = defineRegistry("app.names", { version: 2 }); + const provider = (map: ((names: string[]) => string) | undefined) => + defineFacet((h, self) => { + h.serve(NamesV2, { + compose: (table) => table.map(({ value }) => value), + ...(map === undefined ? {} : { accepts: { 1: { value: map } } }), + }); + self.contribute(NamesV2, "a"); + }); + const host = createHost(noErrors); + await host.load([{ name: "@app/names", facet: provider((names) => names.join(",")) }], context); + const old = await host.registry(NamesV1, context); + expect(old.value).toBe("a"); + await host.reload([{ name: "@app/names", facet: provider((names) => `${names.join("+")}!`) }], context); + expect(old.value).toBe("a!"); + await host.reload([{ name: "@app/names", facet: provider(undefined) }], context); + expect(() => old.value).toThrow("version 1 is not served here"); + await host.close(context); + }); +}); + +describe("reload across hosts, in the app's order (slice 7b §11)", () => { + it("server, then worker, then tab: an old tab is mapped, then waits for its version, then reloads its own code", async () => { + const serveV1 = (h: FacetHost) => + h.serve(Title, () => ({ variant: "v", members: { title: constant("one") } })); + const serveV2 = (text: string, accept: boolean) => (h: FacetHost) => + h.serve( + TitleV2, + () => ({ variant: "v", members: { title: constant({ text }) } }), + accept ? { accepts: { 1: { state: { v: { title: (current) => current.text } } } } } : {}, + ); + const workerSet = (setup: (h: FacetHost) => void) => [ + { + name: "@test/worker", + facet: defineFacet((h) => { + serveNotes(h, PlacedNotes); + setup(h); + }), + }, + ]; + const { tab, server, workers } = await relayed({ worker: serveV1 }); + const old = tab.hold(Title, { session: "S1" }, context); + await transported(); + expect(old.title.value).toBe("one"); + const worker = workers.get("session|S1")!; + // The server installs first: nothing of Title changes there. + await server.reload([{ name: "@test/server", facet: defineFacet(() => {}) }], context); + // The worker installs v2 accepting v1: the old tab is mapped. + await worker.reload(workerSet(serveV2("two", true)), context); + await transported(); + expect(old.state.value).toEqual({ kind: "live" }); + expect(old.title.value).toBe("two"); + // A worker install without the adapter: the old tab waits for its version (a remote failure is its reads' error). + await worker.reload(workerSet(serveV2("three", false)), context); + await transported(); + expect(old.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + }); + // The tab reloads its own code: its new hold (v2) binds; the old one is released. + const fresh = tab.hold(TitleV2, { session: "S1" }, context); + old.release(); + await transported(); + expect(fresh.title.value).toEqual({ text: "three" }); + await server.close(context); + }); +}); + +describe("reload: an abort while staging (fix round 1)", () => { + it("does not commit when the context is aborted by the synchronous setup", async () => { + const T = local("astra.cancel", { version: 1, params: {}, members: { n: int() } }); + const host = createHost({ onError: () => {} }); + const original = defineFacet((h) => h.serve(T, () => ({ n: constant(1) }))); + await host.load([{ name: "@astra/cancel", facet: original }], context); + const ref = await host.get(T, {}, context); + const abort = new AbortController(); + const changed = defineFacet((h) => { + abort.abort(); + h.serve(T, () => ({ n: constant(2) })); + }); + await host + .reload([{ name: "@astra/cancel", facet: changed }], withAbortSignal(abort.signal, context)) + .catch(() => {}); + await flushed(); + const actual = ref.n.value; + ref.release(); + await host.close(context); + expect(actual).toBe(1); + }); +}); diff --git a/packages/chord/test/next/host/members.test.ts b/packages/chord/test/next/host/members.test.ts index 2efc060e0..9c5e40793 100644 --- a/packages/chord/test/next/host/members.test.ts +++ b/packages/chord/test/next/host/members.test.ts @@ -22,10 +22,10 @@ describe("the member check (Q14)", () => { }), ); await expect(host.get(Named, { n: 1 }, context)).rejects.toThrow( - "app.named(1): label: source 1 is not a JSON watchable", + "app.named(1): label: source 1 is not a Chord watchable", ); await expect(host.get(Named, { n: 2 }, context)).rejects.toThrow( - "app.named(2): derive #1: source 0 is not a JSON watchable", + "app.named(2): derive #1: source 0 is not a Chord watchable", ); }); }); diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts index d492240f0..d177122ea 100644 --- a/packages/chord/test/next/host/object.test.ts +++ b/packages/chord/test/next/host/object.test.ts @@ -1,12 +1,24 @@ import { describe, expect, it, vi } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; -import { arr, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { arr, evolve, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { broken, ChordError } from "../../../src/next/data/index.ts"; import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; -import type { Self } from "../../../src/next/host/index.ts"; +import { Attempt } from "../../../src/next/host/attempt.ts"; +import { + createHost, + defineFacet, + type FacetHost, + type FacetSelf, + type Host, + type HostError, + type Self, + type StoreSelf, + untilAborted, +} from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; import { memoryLinkPair } from "../../../src/next/link/index.ts"; import { jsonPart, type Message } from "../../../src/next/link/internal.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant, type Watchable } from "../../../src/next/state/index.ts"; import { changesOf, @@ -26,8 +38,11 @@ import { rejection, relayed, restartable, + SERVER_ADDRESS, + Server, Session, StoreRestartable, + statesOf, storeRestartable, testCodec, transported, @@ -238,7 +253,7 @@ describe("retries", () => { }); }); -describe("dependency recovery (R18)", () => { +describe("parked on dependencies (dependency model §3)", () => { fakeTimers(); const View = local("app.view", { version: 1, params: { id: int() }, members: { n: int() } }); @@ -252,7 +267,7 @@ describe("dependency recovery (R18)", () => { return { times, factory }; } - it("fails a dependent transient with the dependency's cause and retries it at once when it goes live", async () => { + it("parks a dependent of a failed dependency failed { waiting } naming it, with no timer; it goes on when the dependency is live", async () => { const core = flaky((attempt) => attempt === 1); const viewer = view(Core); const { host, errors } = await loaded((h) => { @@ -260,26 +275,28 @@ describe("dependency recovery (R18)", () => { h.serve(View, viewer.factory); }); await vi.advanceTimersByTimeAsync(500); + const held = host.hold(View, { id: 1 }, context); const got = host.get(View, { id: 1 }, context); await flushed(); - const failed = errors.find((error) => error.kind === "activation_failed" && error.object === "app.view(1)"); - expect(failed).toMatchObject({ - failure: { - kind: "transient", - error: { code: "unavailable", message: "app.view(1): dependency app.core failed" }, + expect(held.state.value).toEqual({ + kind: "failed", + cause: { + kind: "waiting", + error: expect.objectContaining({ message: "waiting for app.core: app.core: offline" }), }, + retryAt: undefined, }); - expect(failed?.kind === "activation_failed" && failed.failure.error.cause).toMatchObject({ - message: "app.core: offline", - }); - // The core's retry at 1 s succeeds; the view is retried then, not at the end of its own window (1.5 s). + // A parked attempt is no failure of its own: nothing is reported for the view. + expect(failures(errors)).toEqual(["app.core: transient"]); + // The core's own backoff retries it at 1 s; the view goes on then, its factory not run again. await vi.advanceTimersByTimeAsync(500); expect((await got).n.value).toBe(2); - await vi.advanceTimersByTimeAsync(5000); - expect(viewer.times).toEqual([500, 1000]); + await vi.advanceTimersByTimeAsync(120_000); + expect(viewer.times).toEqual([500]); + held.release(); }); - it("keeps the dependency while failed; releasing the dependent releases it", async () => { + it("keeps its references while parked; a getter rejects with the waiting cause at its deadline; releasing ends the wait", async () => { const svc = flaky(() => true); const viewer = view(Svc); const { host } = await loaded((h) => { @@ -288,81 +305,110 @@ describe("dependency recovery (R18)", () => { }); const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 1500 })); await vi.advanceTimersByTimeAsync(1500); - expect(await got).toEqual({ code: "unavailable", message: "app.view(1): dependency app.svc(1) failed" }); + expect(await got).toEqual({ + code: "unavailable", + message: "waiting for app.svc(1): app.svc(1): offline", + }); await vi.advanceTimersByTimeAsync(10_000); - // Held by the failed view, the dependency was retried at 1 s; released with it at 1.5 s, it is not any more. + // Held by the parked view, the dependency was retried at 1 s; released with it at 1.5 s, it is not any more. expect(svc.times).toEqual([0, 1000]); + expect(viewer.times).toEqual([0]); }); - it("binds a retry's references before releasing the previous attempt's", async () => { - const svc = flaky((attempt) => attempt === 1); - const viewer = view(Svc); + it("shows every change of a dependency: loading, failed, a new cause, loading again, live", async () => { + let cores = 0; + const viewer = view(Core); const { host } = await loaded((h) => { - h.serve(Svc, svc.factory); + h.serve(Core, (_, self) => { + const attempt = ++cores; + self.open( + async () => { + if (attempt < 3) throw new Error(`offline ${attempt}`); + }, + () => {}, + ); + return { n: constant(attempt) }; + }); h.serve(View, viewer.factory); }); - const got = host.get(View, { id: 1 }, context); - await vi.advanceTimersByTimeAsync(1000); - // Held only by the view: released first, the live dependency would have ended and started anew. - expect((await got).n.value).toBe(2); - expect([svc.times, viewer.times]).toEqual([ - [0, 1000], - [0, 1000], - ]); + const held = host.hold(View, { id: 1 }, context); + await flushed(); + expect(held.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.core: app.core: offline 1" } }, + }); + const states = statesOf(await held.state.watch(context)); + await vi.advanceTimersByTimeAsync(10_000); + // The core's retries at 1 s and 3 s: each attempt shows as loading, each failure as a new cause. + expect(states).toEqual(["loading", "failed: waiting for app.core: app.core: offline 2", "loading", "live"]); + expect(viewer.times).toEqual([0]); + held.release(); }); - it("remembers a dependency going live while the dependent activates: the dependent is retried at once (audit)", async () => { - const viewer = view(Core); + it("passes a retry on to its broken dependency, once through a diamond", async () => { + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const Left = local("app.left", { version: 1, params: { id: int() }, members: { n: int() } }); let cores = 0; const { host } = await loaded((h) => { - // Its first factory throws (broken); the second goes live at once, without natives. h.serve(Core, () => { - if (++cores === 1) throw new Error("first factory"); - return { n: constant(2) }; + if (++cores === 1) throw new Error("bug"); + return { n: constant(cores) }; + }); + h.serve(Left, (_, self) => ({ n: self.get(Core, {}).n })); + h.serve(Top, (_, self) => { + const a = self.get(Left, { id: 1 }).n; + const b = self.get(Left, { id: 2 }).n; + return { n: self.derive([a, b], ([x, y]) => x + y) }; }); - h.serve(View, viewer.factory); }); + const top = host.hold(Top, {}, context); await flushed(); - // The view activates on the core's failure while the core is retried and goes live at once. - const dependency = host.hold(Core, {}, context); - const dependent = host.hold(View, { id: 1 }, context); - dependency.retry(); + expect(top.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.left(1): app.core: bug" } }, + }); + await vi.advanceTimersByTimeAsync(60_000); + // A broken dependency is not retried by any timer. + expect(cores).toBe(1); + top.retry(); await flushed(); - expect(dependency.state.value).toEqual({ kind: "live" }); - expect(dependent.state.value).toEqual({ kind: "live" }); - expect(viewer.times).toEqual([0, 0]); - dependent.release(); - dependency.release(); + expect(cores).toBe(2); + expect(top.state.value).toEqual({ kind: "live" }); + expect(top.n.value).toBe(4); + top.release(); }); - it("ignores a trigger while an attempt runs: simultaneous triggers give one attempt", async () => { - const gate = Promise.withResolvers(); - const svc = flaky((attempt) => attempt === 1, gate.promise); - const viewer = view(Svc); - const { host } = await loaded((h) => { - h.serve(Svc, svc.factory); - h.serve(View, viewer.factory); - }); - const got = host.get(View, { id: 1 }, context); - // At 1 s both windows end: the dependency's retry waits at its gate, the view's waits for it. - await vi.advanceTimersByTimeAsync(1000); - gate.resolve(); - expect((await got).n.value).toBe(2); - expect(viewer.times).toEqual([0, 1000]); - }); - - it("retries a dependent of a broken dependency with its backoff, for as long as it is held", async () => { - const viewer = view(Core); + it("walks a diamond on one host once: a shared parked dependency passes a retry on once (audit A5)", async () => { + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const Left = local("app.left", { version: 1, params: { id: int() }, members: { n: int() } }); + const Mid = local("app.mid", { version: 1, params: {}, members: { n: int() } }); + let cores = 0; const { host } = await loaded((h) => { h.serve(Core, () => { - throw new Error("bug"); + if (++cores === 1) throw new Error("bug"); + return { n: constant(cores) }; + }); + h.serve(Mid, (_, self) => ({ n: self.get(Core, {}).n })); + h.serve(Left, (_, self) => ({ n: self.get(Mid, {}).n })); + h.serve(Top, (_, self) => { + const a = self.get(Left, { id: 1 }).n; + const b = self.get(Left, { id: 2 }).n; + return { n: self.derive([a, b], ([x, y]) => x + y) }; }); - h.serve(View, viewer.factory); }); - const got = rejection(host.get(View, { id: 1 }, context, { readyTimeout: 16_000 })); - await vi.advanceTimersByTimeAsync(16_000); - expect(await got).toEqual({ code: "unavailable", message: "app.view(1): dependency app.core failed" }); - expect(viewer.times).toEqual([0, 1000, 3000, 7000, 15000]); + const top = host.hold(Top, {}, context); + await flushed(); + const passed = vi.spyOn(Attempt.prototype, "retryDependencies"); + try { + top.retry(); + // Top, both lefts, and the shared mid once (its second visit through the other left is ignored). + expect(passed).toHaveBeenCalledTimes(4); + } finally { + passed.mockRestore(); + } + await flushed(); + expect([cores, top.n.value]).toEqual([2, 4]); + top.release(); }); it("fails every object of a cycle broken; none is retried", async () => { @@ -515,6 +561,39 @@ describe("GC of failed objects", () => { }); }); +describe("GC of a parked chain", () => { + it("collects a chain parked on a type not served, released from its top: objects, attempts and index entries", async () => { + const Top = local("app.top", { version: 1, params: { id: int() }, members: { n: int() } }); + const Mid = local("app.mid", { version: 1, params: { id: int() }, members: { n: int() } }); + const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + const refs: WeakRef[] = []; + const { host } = await loaded((h) => { + h.serve(Mid, (_, self) => { + refs.push(new WeakRef(self)); + return { n: self.get(Missing, {}).n }; + }); + h.serve(Top, ({ id }, self) => { + refs.push(new WeakRef(self)); + return { n: self.get(Mid, { id }).n }; + }); + }); + const top = host.hold(Top, { id: 1 }, context); + await flushed(); + expect(top.state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { message: "waiting for app.mid(1): app.missing is not served here" }, + }, + }); + top.release(); + await flushed(); + expect(refs).toHaveLength(2); + await expectCollected(...refs); + expect(host).toBeDefined(); + }); +}); + describe("ref.retry()", () => { fakeTimers(); @@ -705,14 +784,19 @@ describe("linger", () => { /** Every object attempted in this file, weakly: the objects of the cycles among them. */ const attempted = instancesOf(ChordObject, "attempt"); -/** On the server (no placement): its activation holds `Far` on the session's worker. */ -const Near = remote("app.near", { version: 1, params: { session: str() }, variants: { v: { state: { n: int() } } } }); +/** On the server: its activation holds `Far` on the session's worker. */ +const Near = remote("app.near", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); /** On the session's worker: its activation holds `Near` back on the server, or, with `local`, `Middle`. */ const Far = remote("app.far", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { v: { state: { n: int() } } }, }); @@ -720,6 +804,7 @@ const Far = remote("app.far", { const Middle = remote("app.middle", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); @@ -730,7 +815,7 @@ const Hub = local("app.hub", { version: 1, params: { session: str() }, members: const Loop = remote("app.loop", { version: 1, params: { here: str(), there: str() }, - placement: ({ here }) => Session({ id: here }), + hostAddress: ({ here }) => Session({ id: here }), variants: { v: { state: { n: int() } } }, }); @@ -910,7 +995,15 @@ function farHold(hold: number): Extract { /** A hold of `Near` for S1, as a worker sends it to the server. */ function nearHold(hold: number): Extract { const { name: type, version, fingerprint } = Near; - return { kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; + return { + kind: "hold", + hold, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; } describe("a failed activation's cleanup (audit)", () => { @@ -1048,7 +1141,9 @@ describe("a live object failing (slice 6a §1.4)", () => { await delivered(); expect(ref.state.value.kind).toBe("failed"); expect([counter!.value, ref.double.value, viewer.tens.value]).toEqual([{ n: 1 }, 2, 20]); - expect(valuesOf(doubles)).toEqual([2]); + // The admitted commit publishes once; then the reference's member holds it (the failure once). A derive over it + // keeps its last value. + expect(valuesOf(doubles)).toEqual([2, { failed: "boom" }]); expect(valuesOf(tens)).toEqual([20]); await host.close(context); await store.close(context); @@ -1104,3 +1199,732 @@ describe("a live object failing (slice 6a §1.4)", () => { expect(lists[1]!.value).toEqual({ items: ["a"] }); }); }); + +/** Reloads `host`'s plugin `@test/app` (as `loaded` names it) with a facet whose setup is `setup`. */ +function reloadApp(host: Host, setup: (host: FacetHost, self: FacetSelf) => void) { + return host.reload([{ name: "@test/app", facet: defineFacet(setup) }], context); +} + +/** A local object with empty params: a counter document, `bump` (which waits for `gate` when given) and `label`. */ +const Pure = local("app.pure", { + version: 1, + params: {}, + members: { n: int(), label: str(), bump: method(undefined, int()) }, +}); + +/** Serves `Pure` labelled `label`; `bump` adds `by`, first awaiting `gates.shift()` when one is queued. */ +function servePure(h: FacetHost, label: string, by: number, gates: Promise[] = []): void { + h.serve(Pure, (_, self) => { + const doc = self.state(defineState("pure", COUNTER)); + return { + n: self.derive([doc], ([d]) => d.n), + label: constant(label), + bump: async (ctx: Context) => { + await gates.shift(); + await doc.change((draft) => { + draft.n += by; + }, ctx); + return doc.value.n; + }, + }; + }); +} + +describe("reload: fates (slice 7b §4)", () => { + it("swaps a pure object in place: same documents and instance id, new code for new calls", async () => { + const { host } = await loaded((h) => servePure(h, "v1", 1)); + const pure = await host.get(Pure, {}, context); + await pure.bump(context); + const objects = instancesOf(ChordObject, "reloadFate"); + const report = await reloadApp(host, (h) => servePure(h, "v2", 10)); + expect(report.objects).toEqual([{ object: "app.pure", kind: "swapped" }]); + expect(objects).toHaveLength(1); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + expect(pure.label.value).toBe("v2"); + expect(await pure.bump(context)).toBe(11); + await host.close(context); + }); + + it("restarts an object with natives: the drain first, its natives closed with reason reload, a new instance", async () => { + const closes: string[] = []; + const serve = (h: FacetHost, label: string, gates: Promise[] = []) => + h.serve(Pure, (_, self) => { + const doc = self.state(defineState("pure", COUNTER)); + self.open( + async () => label, + (value, reason) => void closes.push(`${value} ${reason.kind}`), + ); + return { + n: self.derive([doc], ([d]) => d.n), + label: constant(label), + bump: async (ctx: Context) => { + await gates.shift(); + await doc.change((draft) => { + draft.n += 1; + }, ctx); + closes.push(`bump done on ${label}`); + return doc.value.n; + }, + }; + }); + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => serve(h, "v1", [gate.promise])); + const pure = await host.get(Pure, {}, context); + const running = pure.bump(context); + await flushed(); + const report = await reloadApp(host, (h) => serve(h, "v2")); + expect(report.objects).toEqual([{ object: "app.pure", kind: "restarted" }]); + expect(pure.state.value).toEqual({ kind: "loading" }); + // A new call parks until the new attempt is live. + const parked = pure.bump(context); + await flushed(); + expect(closes).toEqual([]); + gate.resolve(); + expect(await running).toBe(1); + expect(await parked).toBe(2); + expect(closes).toEqual(["bump done on v1", "v1 reload", "bump done on v2"]); + expect(pure.label.value).toBe("v2"); + await host.close(context); + }); + + it("ends the objects of a removed type: its holders wait not_served, and bind again when it is served again", async () => { + const { host, errors } = await loaded((h) => servePure(h, "v1", 1)); + const pure = host.hold(Pure, {}, context); + await flushed(); + await pure.bump(context); + const label = await pure.label.watch(context); + const frames = changesOf(label); + const report = await host.reload([], context); + expect(report).toEqual({ + plugins: { added: [], changed: [], removed: ["@test/app"] }, + failed: [], + objects: [{ object: "app.pure", kind: "ended" }], + }); + expect(pure.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "app.pure is not served here" } }, + }); + // The member keeps its last value while waiting. + expect(pure.label.value).toBe("v1"); + await host.reload([{ name: "@test/app", facet: defineFacet((h) => servePure(h, "v2", 1)) }], context); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + // A new object: in-memory documents went with the old one. + expect(pure.n.value).toBe(0); + await delivered(); + // The removal's failure once, then the new object's value, whole. + expect(valuesOf(frames)).toEqual([{ failed: "app.pure is not served here" }, "v2"]); + expect(failures(errors)).toEqual([]); + await host.close(context); + }); + + it("a live local dependent of a removed type keeps its last value; served again, its derive heals (one frame)", async () => { + const Panel = local("app.panel", { version: 1, params: {}, members: { text: str() } }); + const panel = defineFacet((h) => + h.serve(Panel, (_, self) => { + const pure = self.get(Pure, {}); + return { text: self.derive([pure.label], ([label]) => `panel ${label}`) }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: defineFacet((h) => servePure(h, "v1", 1)) }, + { name: "@test/panel", facet: panel }, + ], + context, + ); + const view = await host.get(Panel, {}, context); + const text = changesOf(await view.text.watch(context)); + await host.reload([{ name: "@test/panel", facet: panel }], context); + expect(view.state.value).toEqual({ kind: "live" }); + expect(view.text.value).toBe("panel v1"); + await host.reload( + [ + { name: "@test/app", facet: defineFacet((h) => servePure(h, "v2", 1)) }, + { name: "@test/panel", facet: panel }, + ], + context, + ); + await flushed(); + expect(view.text.value).toBe("panel v2"); + await delivered(); + expect(valuesOf(text)).toEqual(["panel v2"]); + await host.close(context); + }); + + it("retries a failed object on the new code", async () => { + const { host } = await loaded((h) => + h.serve(Pure, () => { + throw new Error("bug"); + }), + ); + const pure = host.hold(Pure, {}, context); + await flushed(); + expect(pure.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + const report = await reloadApp(host, (h) => servePure(h, "fixed", 1)); + expect(report.objects).toEqual([{ object: "app.pure", kind: "retried" }]); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + expect(pure.label.value).toBe("fixed"); + await host.close(context); + }); + + it("parks a swap whose new factory needs a missing dependency; it goes live when the dependency is served", async () => { + const Dep = local("app.dep", { version: 1, params: {}, members: { name: str() } }); + const { host } = await loaded((h) => servePure(h, "v1", 1)); + const pure = await host.get(Pure, {}, context); + const needsDep = (h: FacetHost) => + h.serve(Pure, (_, self) => { + const dep = self.get(Dep, {}); + return { + n: constant(0), + label: self.derive([dep.name], ([name]) => `with ${name}`), + bump: async () => 0, + }; + }); + const report = await reloadApp(host, needsDep); + expect(report.objects).toEqual([ + { + object: "app.pure", + kind: "parked", + cause: { kind: "waiting", error: expect.objectContaining({ message: expect.stringContaining("app.dep") }) }, + }, + ]); + expect(pure.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(await rejection(pure.bump(context))).toMatchObject({ code: "unavailable" }); + await host.reload( + [ + { name: "@test/app", facet: defineFacet(needsDep) }, + { name: "@test/dep", facet: defineFacet((h) => h.serve(Dep, () => ({ name: constant("dep") }))) }, + ], + context, + ); + await flushed(); + expect(pure.state.value).toEqual({ kind: "live" }); + expect(pure.label.value).toBe("with dep"); + await host.close(context); + }); +}); + +describe("reload: the pure swap (slice 7b §5)", () => { + it("lets a running call finish on the old attempt's handles; a new call waits at the gate for the new code", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => servePure(h, "v1", 1, [gate.promise])); + const pure = await host.get(Pure, {}, context); + const old = pure.bump(context); + await flushed(); + await reloadApp(host, (h) => servePure(h, "v2", 10)); + // The old call commits through its old handle after the swap. + gate.resolve(); + expect(await old).toBe(1); + expect(await pure.bump(context)).toBe(11); + await host.close(context); + }); + + it("an old call touching a document whose version rose throws once (R15); the document is migrated", async () => { + const V1 = defineState("doc", COUNTER); + const V2 = defineState("doc", { + schema: obj({ n: int(), unit: str() }), + version: 2, + initial: () => ({ n: 0, unit: "" }), + migrate: (old) => ({ n: old.n as number, unit: "pcs" }), + }); + const gate = Promise.withResolvers(); + const serve = (h: FacetHost, definition: typeof V1 | typeof V2, gates: Promise[]) => + h.serve(Pure, (_, self) => { + const doc = self.state(definition as typeof V1); + return { + n: self.derive([doc], ([d]) => d.n), + label: self.derive([doc], ([d]) => JSON.stringify(d)), + bump: async (ctx: Context) => { + await gates.shift(); + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }; + }); + const { host } = await loaded((h) => serve(h, V1, [Promise.resolve(), gate.promise])); + const pure = await host.get(Pure, {}, context); + await pure.bump(context); + const old = pure.bump(context); + await flushed(); + await reloadApp(host, (h) => serve(h, V2, [])); + await flushed(); + expect(pure.label.value).toBe('{"n":1,"unit":"pcs"}'); + gate.resolve(); + expect(await rejection(old)).toMatchObject({ code: "released", reason: "version" }); + expect(await pure.bump(context)).toBe(2); + await host.close(context); + }); + + it("binds the new attempt's requests before releasing the old one's: a dependency held only by it never ends", async () => { + const Dep = local("app.dep", { + version: 1, + params: { id: int() }, + members: { n: int(), set: method(obj({ n: int() })) }, + }); + const dep = defineFacet((h) => + h.serve(Dep, (_, self) => { + const doc = self.state(defineState("dep", COUNTER)); + return { + n: self.derive([doc], ([d]) => d.n), + set: async ({ n }: { n: number }, ctx: Context) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }; + }), + ); + const app = (label: string) => + defineFacet((h) => + h.serve(Pure, (_, self) => { + const held = self.get(Dep, { id: 1 }); + return { + n: self.derive([held.n], ([n]) => n), + label: constant(label), + bump: async (ctx: Context) => { + await held.set({ n: 7 }, ctx); + return 7; + }, + }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: app("v1") }, + { name: "@test/dep", facet: dep }, + ], + context, + ); + const pure = await host.get(Pure, {}, context); + await pure.bump(context); + await host.reload( + [ + { name: "@test/app", facet: app("v2") }, + { name: "@test/dep", facet: dep }, + ], + context, + ); + await flushed(); + // The dependency's in-memory document is still there: its object never ended in between. + expect(pure.label.value).toBe("v2"); + expect(pure.n.value).toBe(7); + await host.close(context); + }); + + it("re-binds members by name (one frame from the last value) and rebuilds internal derives", async () => { + const { host } = await loaded((h) => servePure(h, "v1", 1)); + const pure = await host.get(Pure, {}, context); + const label = changesOf(await pure.label.watch(context)); + const n = changesOf(await pure.n.watch(context)); + await pure.bump(context); + await reloadApp(host, (h) => servePure(h, "v2", 1)); + await delivered(); + expect(label).toEqual([{ kind: "value", value: "v2", ops: [["r", "v2"]], retired: false }]); + // Its derive over the same document gave the same value: no frame besides the commit's. + expect(valuesOf(n)).toEqual([1]); + await host.close(context); + }); + + it("hands its routines over: the old ones aborted, the new ones start after the old calls finished", async () => { + const events: string[] = []; + const release = Promise.withResolvers(); + const serve = (h: FacetHost, label: string) => + h.serve(Pure, (_, self) => { + self.onLive(async (ctx) => { + events.push(`${label} started`); + await untilAborted(ctx); + if (label === "v1") await release.promise; + events.push(`${label} stopped`); + }); + return { n: constant(0), label: constant(label), bump: async () => 0 }; + }); + const { host } = await loaded((h) => serve(h, "v1")); + await host.get(Pure, {}, context); + await flushed(); + await reloadApp(host, (h) => serve(h, "v2")); + await flushed(); + expect(events).toEqual(["v1 started"]); + release.resolve(); + await flushed(); + expect(events).toEqual(["v1 started", "v1 stopped", "v2 started"]); + await host.close(context); + }); + + it("starts a writable at its new initial value and replaces the contributions in one batch", async () => { + const Tools = defineRegistry("app.tools", { version: 1 }); + const compositions: string[][] = []; + const registry = defineFacet((h) => + h.serve(Tools, { + compose: (table) => { + const value = table.flatMap((entry) => entry.value); + compositions.push(value); + return value; + }, + }), + ); + const app = (tool: string) => + defineFacet((h) => + h.serve(Pure, (_, own) => { + const tools = own.writable([tool]); + (own as unknown as StoreSelf).contribute(Tools, tools); + return { n: constant(0), label: constant(tool), bump: async () => tools.value.length }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/app", facet: app("a") }, + { name: "@test/tools", facet: registry }, + ], + context, + ); + const tools = await host.registry(Tools, context); + expect(tools.value).toEqual(["a"]); + compositions.length = 0; + await host.reload( + [ + { name: "@test/app", facet: app("b") }, + { name: "@test/tools", facet: registry }, + ], + context, + ); + await flushed(); + expect(tools.value).toEqual(["b"]); + // One composition with the new table, never one without a contribution in between. + expect(compositions).toEqual([["b"]]); + await host.close(context); + }); +}); + +describe("reload: a stuck routine call past the handover", () => { + fakeTimers(); + + it("an old routine call stuck past the handover: drain_timeout, the old attempt sealed, its commit refused (audit A12)", async () => { + const stuck = Promise.withResolvers(); + let late: Promise | undefined; + const errors: HostError[] = []; + const serve = (h: FacetHost, label: string) => + h.serve(Pure, (_, self) => { + const doc = self.state(defineState("pure", COUNTER)); + self.onLive(async (ctx) => { + if (label !== "v1") return; + await stuck.promise; + late = rejection( + self.tx(async (tx) => { + (await tx.draft(doc)).n = 99; + }, ctx), + ); + }); + return { n: self.derive([doc], ([d]) => d.n), label: constant(label), bump: async () => 0 }; + }); + const host = createHost({ closeTimeout: 1_000, onError: (error) => errors.push(error) }); + await host.load([{ name: "@test/app", facet: defineFacet((h) => serve(h, "v1")) }], context); + const pure = await host.get(Pure, {}, context); + await reloadApp(host, (h) => serve(h, "v2")); + await vi.advanceTimersByTimeAsync(1_000); + expect(errors.map(({ kind }) => kind)).toContain("drain_timeout"); + stuck.resolve(); + await vi.advanceTimersByTimeAsync(0); + expect(await late).toMatchObject({ code: "released" }); + expect(pure.n.value).toBe(0); + await host.close(context); + }); +}); + +describe("reload: GC", () => { + const attempts = instancesOf(Attempt, "activate"); + + it("collects a swapped-out attempt and its closures once its calls settled", async () => { + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => servePure(h, "v1", 1, [gate.promise])); + const pure = await host.get(Pure, {}, context); + const old = pure.bump(context); + const from = attempts.length - 1; + await reloadApp(host, (h) => servePure(h, "v2", 1)); + gate.resolve(); + await old; + await flushed(); + await expectCollected(attempts[from]!); + expect(pure.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("collects a removed type's objects, its provider, its registry entries and the old facet's code", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const objects = instancesOf(ChordObject, "remove"); + const host = createHost({ onError: () => {} }); + /** Loads the plugin; only the host keeps its facet. */ + const load = (): [WeakRef, Promise] => { + const made = defineFacet((h, self) => { + servePure(h, "v1", 1); + h.serve(Names, { compose: (table) => table.map(({ value }) => value) }); + self.contribute(Names, "x"); + }); + return [new WeakRef(made), host.load([{ name: "@test/app", facet: made }], context)]; + }; + const [facet, loading] = load(); + await loading; + await host.get(Pure, {}, context).then((pure) => pure.release()); + await host.reload([], context); + await flushed(); + expect(objects).toHaveLength(1); + await expectCollected(objects[0]!, facet); + await host.close(context); + }); +}); + +describe("reload: a parked attempt's handover (fix round 1)", () => { + it("keeps a dependency held only by a parked attempt alive through reload", async () => { + const A = local("astra.a", { version: 1, params: {}, members: {} }); + const Dep = local("astra.dep", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("astra.missing", { version: 1, params: {}, members: {} }); + let opens = 0; + let closes = 0; + const host = createHost({ onError: () => {} }); + const depFacet = defineFacet((h) => + h.serve(Dep, (_, self) => { + self.open( + async () => { + opens++; + }, + () => { + closes++; + }, + ); + return {}; + }), + ); + const aFacet = () => + defineFacet((h) => + h.serve(A, (_, self) => { + self.get(Dep, { id: 1 }); + self.get(Missing, {}); + return {}; + }), + ); + await host.load( + [ + { name: "@astra/dep", facet: depFacet }, + { name: "@astra/a", facet: aFacet() }, + ], + context, + ); + await flushed(); + expect(opens).toBe(1); + await host.reload( + [ + { name: "@astra/dep", facet: depFacet }, + { name: "@astra/a", facet: aFacet() }, + ], + context, + ); + await flushed(); + const counts = { opens, closes }; + await host.close(context); + expect(counts).toEqual({ opens: 1, closes: 0 }); + }); + + it("binds replacement requests before releasing the parked predecessor's", async () => { + const A = local("sol.parked", { version: 1, params: {}, members: {} }); + const B = local("sol.native", { version: 1, params: { id: int() }, members: {} }); + const Missing = local("sol.missing", { version: 1, params: {}, members: {} }); + let opens = 0; + let closes = 0; + const b = defineFacet((h) => + h.serve(B, (_, self) => { + self.open( + async () => ++opens, + () => { + closes++; + }, + ); + return {}; + }), + ); + const a = () => + defineFacet((h) => + h.serve(A, (_, self) => { + self.get(B, { id: 1 }); + self.get(Missing, {}); + return {}; + }), + ); + const host = createHost({ onError: () => {} }); + try { + await host.load( + [ + { name: "@sol/b", facet: b }, + { name: "@sol/a", facet: a() }, + ], + context, + ); + expect({ opens, closes }).toEqual({ opens: 1, closes: 0 }); + await host.reload( + [ + { name: "@sol/b", facet: b }, + { name: "@sol/a", facet: a() }, + ], + context, + ); + await delivered(); + expect({ opens, closes }).toEqual({ opens: 1, closes: 0 }); + } finally { + await host.close(context); + } + }); +}); + +describe("reload: swaps one after another (fix round 1)", () => { + it("a second swap does not wait for the first swap's old call; that call still finishes on its old code", async () => { + const gate = Promise.withResolvers(); + const { host, errors } = await loaded((h) => servePure(h, "v1", 1, [gate.promise]), { closeTimeout: 60_000 }); + const pure = await host.get(Pure, {}, context); + const old = pure.bump(context); + await flushed(); + await reloadApp(host, (h) => servePure(h, "v2", 10)); + let resolved = false; + const second = reloadApp(host, (h) => servePure(h, "v3", 100)).then(() => { + resolved = true; + }); + await flushed(); + // Not held up by v1's call (Fable, Opus F1): resolved within a few turns, the gate open for new calls. + expect(resolved).toBe(true); + await second; + expect(pure.label.value).toBe("v3"); + gate.resolve(); + expect(await old).toBe(1); + expect(await pure.bump(context)).toBe(101); + expect(errors).toEqual([]); + await host.close(context); + }); +}); + +describe("reload: a reference detaching from a parked swap's old instance (Opus G1)", () => { + it("follows the old instance while its call drains, then keeps its last value with the failure once", async () => { + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => servePure(h, "v1", 1, [gate.promise]), { closeTimeout: 60_000 }); + const pure = await host.get(Pure, {}, context); + const calls = changesOf(await pure.n.watch(context)); + const bumping = pure.bump(context); + await flushed(); + const parked = defineFacet((h) => + h.serve(Pure, (_, self) => { + self.get(Missing, {}); + return { n: constant(9), label: constant("v2"), bump: async () => 0 }; + }), + ); + const report = await host.reload([{ name: "@test/app", facet: parked }], context); + expect(report.objects[0]).toMatchObject({ kind: "parked" }); + expect(pure.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + // The old instance's call still commits: the reference follows it until it retires. + gate.resolve(); + expect(await bumping).toBe(1); + await delivered(); + expect([pure.n.value, calls.map((call) => call.kind), valuesOf(calls)[0]]).toEqual([1, ["value", "failed"], 1]); + await host.close(context); + }); +}); + +describe("a live failure: an older version's mapping follows the admitted commits (final review)", () => { + it("stops an older reference after its instance's admitted final commit published (Astra, G1)", async () => { + const V1 = local("app.admitted", { version: 1, params: {}, members: { n: int() } }); + const V2 = evolve(V1, (e) => e.state("n", obj({ n: int() }))); + const State = defineState("state", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const store = await memoryStore(); + let own: SingletonHandle<{ n: number }> | undefined; + let self: StoreSelf | undefined; + const facet = defineFacet((h) => + h.serve( + V2, + (_, declared) => { + self = declared; + own = declared.state(State); + return { n: own }; + }, + { accepts: { 1: { state: { n: (value) => value.n } } } }, + ), + ); + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "@app/admitted", facet }], context); + const old = await host.get(V1, {}, context); + const current = await host.get(V2, {}, context); + const calls = changesOf(await old.n.watch(context)); + // A commit admitted before the failure: v1's mapping and v2's member both get it, then the failure. + const held = holdNextCommit(store); + const change = own!.change((draft) => { + draft.n = 5; + }, context); + await held.entered; + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + held.release(); + await change; + await delivered(); + expect({ old: old.n.value, current: current.n.value, calls: valuesOf(calls) }).toEqual({ + old: 5, + current: { n: 5 }, + calls: [5, { failed: "boom" }], + }); + old.release(); + current.release(); + await host.close(context); + await store.close(context); + }); +}); + +describe("a native restart: references take their last values after the admitted commits (final review)", () => { + it("keeps the value of a commit admitted before the restart's seal while the new instance opens (Sol, G1)", async () => { + const Type = local("app.restart-admitted", { version: 1, params: {}, members: { x: obj({ n: int() }) } }); + const Doc = defineState("doc", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 0 }), + persist: true, + checkpointWhen: () => false, + }); + const facet = (open: () => Promise) => + defineFacet((h) => + h.serve(Type, (_, self) => { + counter = self.state(Doc); + self.open(open, () => {}); + return { x: counter }; + }), + ); + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const opening = Promise.withResolvers(); + let counter: SingletonHandle<{ n: number }> | undefined; + await host.load([{ name: "@app/restart", facet: facet(async () => 0) }], context); + const ref = await host.get(Type, {}, context); + // A commit admitted to the store (no running call's promise), then a restart whose new native stays opening. + const held = holdNextCommit(store); + const outgoing = counter!; + const change = outgoing.change((draft) => { + draft.n = 1; + }, context); + await held.entered; + await host.reload([{ name: "@app/restart", facet: facet(() => opening.promise) }], context); + await delivered(); + held.release(); + await change; + await delivered(); + expect([ref.state.value.kind, ref.x.value]).toEqual(["loading", { n: 1 }]); + opening.resolve(1); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/host/presence.test.ts b/packages/chord/test/next/host/presence.test.ts index 6260f9360..a43b523bf 100644 --- a/packages/chord/test/next/host/presence.test.ts +++ b/packages/chord/test/next/host/presence.test.ts @@ -22,6 +22,8 @@ import { PlacedNotes, RawPeer, relayed, + SERVER_ADDRESS, + Server, serveNotes, type TestPrincipal, testCodec, @@ -84,9 +86,10 @@ describe("host.references", () => { const Shown = remote("app.shown", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { v: { state: { here: int() } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( [ { @@ -130,9 +133,10 @@ describe("host.references", () => { const Viewed = remote("app.viewed", { version: 1, params: { session: str() }, + hostAddress: () => Server(), variants: { v: { state: { count: int(), viewers: int() } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((host) => { serveNotes(host); host.serve(Viewed, ({ session }, _principal, self) => { @@ -151,7 +155,15 @@ describe("host.references", () => { server.accept(end, { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); const { name: type, version, fingerprint } = Viewed; - peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: jsonPart({ session: "S1" }) }); + peer.send({ + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); return peer; }; const drop = { armed: false }; @@ -176,9 +188,10 @@ describe("host.references", () => { const Viewed = remote("app.viewed", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { viewers: int() } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((host) => { host.serve( Viewed, @@ -195,13 +208,13 @@ describe("host.references", () => { server.accept(closingOnSend(a, drop), { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); const { name: type, version, fingerprint } = Viewed; - peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await peer.next("bound"); peer.send({ kind: "ack" }); await transported(); // The second hold's presence entry changes the first's state: sending it closes the link. drop.armed = true; - peer.send({ kind: "hold", hold: 1, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: 1, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await transported(); expect(peer.closed).toBe(true); expect(server.references(Viewed).value).toEqual([]); @@ -210,8 +223,13 @@ describe("host.references", () => { }); it("completes a release's bookkeeping before its departure publishes: a link closing then leaves no lingering", async () => { - const Viewed = remote("app.viewed", { version: 1, params: {}, variants: { v: { state: { viewers: int() } } } }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const Viewed = remote("app.viewed", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { viewers: int() } } }, + }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((host) => { host.serve( Viewed, @@ -229,7 +247,7 @@ describe("host.references", () => { const peer = new RawPeer(b); const { name: type, version, fingerprint } = Viewed; for (const hold of [0, 1]) { - peer.send({ kind: "hold", hold, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await peer.next("bound"); peer.send({ kind: "ack" }); } @@ -248,6 +266,7 @@ describe("host.references", () => { it("removes a bulk departure with one list and one splice op per entry, in one frame (a revoke of many holds)", async () => { const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: (principal) => (principal.kind === "guest" ? [{ session: principal.session }] : []), onError: () => {}, @@ -258,7 +277,15 @@ describe("host.references", () => { const peer = new RawPeer(b); const { name: type, version, fingerprint } = Notes; for (let hold = 0; hold < 4; hold++) { - peer.send({ kind: "hold", hold, type, version, fingerprint, params: jsonPart({ session: "S1" }) }); + peer.send({ + kind: "hold", + hold, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }); } await transported(); const references = server.references(Notes); @@ -285,7 +312,12 @@ describe("host.references", () => { }); it("has no entries for lingering objects", async () => { - const Kept = remote("app.kept", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Kept = remote("app.kept", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { server, client } = await linked((h) => h.serve(Kept, () => ({ variant: "v", members: { n: constant(1) } }), { linger: 60_000 }), ); diff --git a/packages/chord/test/next/host/ref-state.test.ts b/packages/chord/test/next/host/ref-state.test.ts index d7ab08c47..360a7bd3e 100644 --- a/packages/chord/test/next/host/ref-state.test.ts +++ b/packages/chord/test/next/host/ref-state.test.ts @@ -181,7 +181,8 @@ forEachCodec(() => { await transported(); expect(notes.state.value).toMatchObject({ kind: "failed", retryAt: undefined }); expect(notes.variant).toBe("editor"); - expect(() => notes.count.value).toThrow("offline"); + // Its replica's last value stays readable while the hold is failed (rule 5). + expect(notes.count.value).toBe(1); notes.release(); expect(() => notes.variant).toThrow("app.notes: released"); }); diff --git a/packages/chord/test/next/host/keyed-members.test.ts b/packages/chord/test/next/host/reference-keys.test.ts similarity index 58% rename from packages/chord/test/next/host/keyed-members.test.ts rename to packages/chord/test/next/host/reference-keys.test.ts index d2570dd56..cdef55686 100644 --- a/packages/chord/test/next/host/keyed-members.test.ts +++ b/packages/chord/test/next/host/reference-keys.test.ts @@ -1,18 +1,21 @@ -// Tests of keyed members across attempts (slice 6a §1.2, §1.4, §12): the keys references hold on an object's keyed -// members, frozen while it is failed, re-bound by name (loaded first) when its next attempt goes live, released once. +// Tests of the keys a reference holds (slice 6a §12; decision "References are the only indirection" (4)): its key +// watches and factory pins of an object's keyed members, holding their last values while it is failed (one `failed` +// call), pinned again on its next instance once that is live (one frame), released once. -import { describe, expect, it } from "vitest"; -import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; -import { int, keyed, local, obj, str } from "../../../src/next/contracts/index.ts"; +import { describe, expect, it, vi } from "vitest"; +import { type Context, BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, keyed, local, method, obj, str } from "../../../src/next/contracts/index.ts"; import { ChordError } from "../../../src/next/data/index.ts"; import { defineState, type KeyedHandle } from "../../../src/next/documents/index.ts"; -import type { Self } from "../../../src/next/host/index.ts"; +import { createHost, defineFacet, type Self } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; import { changesOf, delivered, expectCollected, failLive, + flushed, + holdNextCommit, instancesOf, loaded, memoryStore, @@ -70,7 +73,7 @@ async function served(which: () => string) { } describe("a failed object's held keys", () => { - it("keep their last values and get no frames (at, watch, a derive over at); the provider goes on", async () => { + it("keep their last values and get the failure once (at, watch, a derive over at); the provider goes on", async () => { const { host, a, exposer, viewer, set, fail } = await served(() => "a"); const values = changesOf(await viewer.value.watch(context)); const keyWatch = (await exposer.notes.watch("k", context))!; @@ -81,14 +84,14 @@ describe("a failed object's held keys", () => { await delivered(); expect(exposer.state.value.kind).toBe("failed"); expect([viewer.value.value, viewer.double.value, keyWatch.value]).toEqual([{ n: 1 }, 2, { n: 1 }]); - expect([values, keyed]).toEqual([[], []]); + expect([values, keyed].map((calls) => calls.map((call) => call.kind))).toEqual([["failed"], ["failed"]]); expect(valuesOf(direct)).toEqual([{ n: 3 }]); await host.close(context); }); }); describe("re-binding when a new attempt goes live", () => { - it("follows the new attempt's member of the same name (one frame); the old provider's pin is released", async () => { + it("pins again on the new instance's member of the same name (one frame); the old provider's pin is released", async () => { let which = "a"; const { host, exposer, viewer, set, fail, providers } = await served(() => which); const values = changesOf(await viewer.value.watch(context)); @@ -100,6 +103,8 @@ describe("re-binding when a new attempt goes live", () => { ?.pins; expect(pins("a")).toBe(1); fail(); + // Its keys follow the failed instance until its admitted commits published (audit A10), then show the failure. + await delivered(); which = "b"; exposer.retry(); (await host.get(Exposer, {}, context)).release(); @@ -109,13 +114,15 @@ describe("re-binding when a new attempt goes live", () => { await set("b", 4); await set("a", 5); await delivered(); - expect(valuesOf(values)).toEqual([{ n: 2 }, { n: 4 }]); - expect(valuesOf(keyed)).toEqual([{ n: 2 }, { n: 4 }]); + // The failure once, then whole values (the first after a failure is `[["r", value]]`). + expect(valuesOf(values)).toEqual([{ failed: "boom" }, { n: 2 }, { n: 4 }]); + expect(valuesOf(keyed)).toEqual([{ failed: "boom" }, { n: 2 }, { n: 4 }]); + // A derive over the key reads its last value meanwhile: it does not fail. expect(valuesOf(doubles)).toEqual([4, 8]); await host.close(context); }); - it("loads a cold persisted key of the new member before the new attempt goes live", async () => { + it("loads a cold persisted key of the new member once the new instance is live (the last value meanwhile)", async () => { const Stored = defineState("notes", { schema: Schema, version: 1, @@ -166,7 +173,8 @@ describe("re-binding when a new attempt goes live", () => { failLive(attempted, exposers[0]!, new ChordError({ code: "unavailable", message: "boom" })); exposer.retry(); (await host.get(Exposer, {}, context)).release(); - expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 10 }, 20]); + // The key loads from the store after the new instance went live: one frame then. + await vi.waitFor(() => expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 10 }, 20])); await host.close(context); await store.close(context); }); @@ -291,7 +299,8 @@ describe("a key whose first pin still loads", () => { await delivered(); expect([viewer.value.value, viewer.double.value]).toEqual([{ n: 2 }, 4]); expect([pinsOf(providers.a!), pinsOf(providers.b!)]).toEqual([undefined, 1]); - expect(errors.filter((error) => error.kind === "activation_failed").length).toBe(1); + // The pin's failed load parked the viewer (a dependency failure is not its own): nothing failed it. + expect(errors.filter((error) => error.kind === "activation_failed").length).toBe(0); const values = changesOf(await viewer.value.watch(context)); await providers.a!.change( "k", @@ -312,10 +321,11 @@ describe("a key whose first pin still loads", () => { await close(); }); - it("the object fails and stays failed: the consumer's activation fails transient, never live on it", async () => { - const { host, errors, exposer, providers, hold, gate, entered, fail, close } = await cold(() => "a"); + it("the object fails and stays failed: the consumer parks failed { waiting } on it, never live on it", async () => { + const { host, exposer, providers, hold, gate, entered, fail, close } = await cold(() => "a"); hold(); const abort = new AbortController(); + const viewer = host.hold(Viewer, { id: 1 }, context); const loading = host.get(Viewer, { id: 1 }, withAbortSignal(abort.signal, context)); await entered.promise; fail(); @@ -323,8 +333,8 @@ describe("a key whose first pin still loads", () => { await delivered(); await new Promise((resolve) => setTimeout(resolve, 10)); expect(exposer.state.value.kind).toBe("failed"); - const failure = errors.find((error) => error.kind === "activation_failed"); - expect(failure).toMatchObject({ failure: { kind: "transient" } }); + expect(viewer.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + viewer.release(); abort.abort(); await expect(loading).rejects.toThrow(); expect(pinsOf(providers.a!)).toBeUndefined(); @@ -375,7 +385,7 @@ describe("a keyed watch whose object closes while it is set up", () => { }); describe("chains and releases", () => { - it("a key re-exposed through two objects freezes with the middle one and follows its next attempt", async () => { + it("a key re-exposed through two objects holds with the middle one (the failure once) and follows its next instance", async () => { const Outer = local("app.outer", { version: 1, params: {}, members: { notes: keyed(Schema) } }); const OuterViewer = local("app.outer-viewer", { version: 1, params: {}, members: { value: Schema } }); let which = "a"; @@ -417,13 +427,13 @@ describe("chains and releases", () => { context, ); await delivered(); - expect([viewer.value.value, values]).toEqual([{ n: 1 }, []]); + expect([viewer.value.value, values.map((call) => call.kind)]).toEqual([{ n: 1 }, ["failed"]]); which = "b"; middle.retry(); (await host.get(Exposer, {}, context)).release(); await delivered(); expect(viewer.value.value).toEqual({ n: 2 }); - expect(valuesOf(values)).toEqual([{ n: 2 }]); + expect(valuesOf(values)).toEqual([{ failed: "boom" }, { n: 2 }]); await host.close(context); }); @@ -450,3 +460,245 @@ describe("chains and releases", () => { await expectCollected(...refs); }); }); + +describe("a key whose first pin still loads at startup (scenario 20, Astra)", () => { + it("keeps the first pass pending during an otherwise-ready dependency's delayed keyed pin", async () => { + const Data = defineState("app.pin-data", { + schema: Schema, + version: 1, + keyed: true, + initial: () => ({ n: 4 }), + }); + const Store = local("app.pin-store", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + const Reader = local("app.pin-reader", { version: 1, params: {}, members: { value: Schema } }); + const gate = Promise.withResolvers(); + const original = ChordObject.prototype.pinKey; + const spy = vi.spyOn(ChordObject.prototype, "pinKey").mockImplementation(async function ( + this: ChordObject, + member, + key, + version, + ctx, + ) { + await gate.promise; + return original.call(this, member, key, version, ctx); + }); + const host = createHost({ onError: () => {} }); + let done = false; + const loading = host + .load( + [ + { + name: "@test/pin", + facet: defineFacet((h) => { + h.serve(Store, (_, self) => ({ notes: self.state(Data) })); + h.serve(Reader, (_, self) => ({ value: self.get(Store, {}).notes.at("k") })); + }), + }, + ], + context, + ) + .then(() => { + done = true; + }); + await flushed(); + const premature = done; + gate.resolve(); + await loading; + spy.mockRestore(); + const ref = await host.get(Reader, {}, context); + expect(ref.value.value).toEqual({ n: 4 }); + await host.close(context); + expect(premature).toBe(false); + }); +}); + +describe("a released key leaves its document (final review)", () => { + it("collects a reference whose key watch stopped while another reference pins the same key (Astra)", async () => { + const Rows = defineState("rows", { + version: 1, + schema: obj({ n: int() }), + initial: () => ({ n: 1 }), + keyed: true, + }); + const Type = local("app.key-gc", { + version: 1, + params: {}, + members: { rows: keyed(obj({ n: int() })), set: method() }, + }); + const facet = defineFacet((h) => + h.serve(Type, (_, self) => { + const rows = self.state(Rows); + const set = async (ctx: Context) => { + await rows.change( + "a", + (draft) => { + draft.n = 2; + }, + ctx, + ); + return null; + }; + return { rows, set }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/key-gc", facet }], context); + const keeper = await host.get(Type, {}, context); + await keeper.set(context); + const kept = (await keeper.rows.watch("a", context))!; + kept.start(async () => {}); + const released = await (async () => { + const ref = await host.get(Type, {}, context); + const watch = (await ref.rows.watch("a", context))!; + watch.start(async () => {}); + await watch.stop(); + ref.release(); + return new WeakRef(ref); + })(); + await flushed(); + await expectCollected(released); + await kept.stop(); + keeper.release(); + await host.close(context); + }); +}); + +describe("keys across a restart and a failed re-pin (Astra's coverage note)", () => { + const Keys = local("app.keys", { version: 1, params: {}, members: { notes: keyed(Schema) } }); + + /** Serves `Keys` over the persisted `Stored` (its handle kept in `handles`), with a native opened by `open`. */ + const serveKeys = (handles: KeyedHandle[], open: () => Promise) => + defineFacet((h) => + h.serve(Keys, (_, self) => { + self.open(open, () => {}); + const notes = self.state(Stored); + handles.push(notes); + return { notes }; + }), + ); + + it("a commit admitted to a watched key, landing after a restart's drain, is in the key's last value", async () => { + const store = await memoryStore(); + const handles: KeyedHandle[] = []; + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "@test/app", facet: serveKeys(handles, async () => {}) }], context); + const keys = await host.get(Keys, {}, context); + const change = (n: number) => + handles.at(-1)!.change( + "k", + (draft) => { + draft.n = n; + }, + context, + ); + await change(1); + const watch = (await keys.notes.watch("k", context))!; + const calls = changesOf(watch); + const held = holdNextCommit(store); + const admitted = change(5); + await held.entered; + // The new code's native stays opening: the new instance is not live while the key holds its last value. + const opening = Promise.withResolvers(); + const reloading = host.reload([{ name: "@test/app", facet: serveKeys(handles, () => opening.promise) }], context); + await delivered(); + held.release(); + await admitted; + await reloading; + await delivered(); + expect([watch.value, valuesOf(calls)]).toEqual([{ n: 5 }, [{ n: 5 }]]); + opening.resolve(); + await vi.waitFor(() => expect(keys.state.value).toEqual({ kind: "live" })); + expect(watch.value).toEqual({ n: 5 }); + await host.close(context); + await store.close(context); + }); + + it("a key's re-pin on a new instance that fails to load keeps its last value, one failed call; the next pin heals it", async () => { + const store = await memoryStore(); + const seeded: KeyedHandle[] = []; + const seed = createHost({ store, onError: () => {} }); + await seed.load([{ name: "@test/app", facet: serveKeys(seeded, async () => {}) }], context); + await seed.get(Keys, {}, context); + await seeded[0]!.change( + "k", + (draft) => { + draft.n = 1; + }, + context, + ); + await seed.close(context); + // A new host: `Keys`' key is in the store only; the exposer re-exposes provider `b` (in memory) first. + const handles: KeyedHandle[] = []; + let b: KeyedHandle | undefined; + const providers = defineFacet((h) => { + h.serve(Keys, (_, self) => { + const notes = self.state(Stored); + handles.push(notes); + return { notes }; + }); + h.serve(Provider, (_, self) => { + b = self.state(Notes); + return { notes: b }; + }); + }); + const exposing = (which: "keys" | "b") => + defineFacet((h) => + h.serve(Exposer, (_, self) => { + self.state(Own); + const notes = which === "keys" ? self.get(Keys, {}).notes : self.get(Provider, { id: "b" }).notes; + return { notes }; + }), + ); + const host = createHost({ store, onError: () => {} }); + await host.load( + [ + { name: "@test/providers", facet: providers }, + { name: "@test/exposer", facet: exposing("b") }, + ], + context, + ); + await host.get(Provider, { id: "b" }, context); + await b!.change( + "k", + (draft) => { + draft.n = 2; + }, + context, + ); + const exposer = await host.get(Exposer, {}, context); + const watch = (await exposer.notes.watch("k", context))!; + const calls = changesOf(watch); + // The cold provider's next read of the key fails once. + const persistence = (handles[0] as unknown as { slot: { persistence: { read(...args: unknown[]): unknown } } }) + .slot.persistence; + const read = persistence.read.bind(persistence); + persistence.read = async () => { + persistence.read = read; + throw new Error("disk"); + }; + // A swap in place: the new instance re-exposes the cold provider's notes; the key's pin there fails to load. + await host.reload( + [ + { name: "@test/providers", facet: providers }, + { name: "@test/exposer", facet: exposing("keys") }, + ], + context, + ); + await flushed(); + expect(exposer.state.value).toEqual({ kind: "live" }); + expect([watch.value, valuesOf(calls)]).toEqual([{ n: 2 }, [{ failed: expect.stringContaining("disk") }]]); + // The next instance pins it again, loaded this time: one frame heals it. + await host.reload( + [ + { name: "@test/providers", facet: providers }, + { name: "@test/exposer", facet: exposing("keys") }, + ], + context, + ); + await vi.waitFor(() => expect(watch.value).toEqual({ n: 1 })); + expect(valuesOf(calls)).toEqual([{ failed: expect.stringContaining("disk") }, { n: 1 }]); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/host/member-source.test.ts b/packages/chord/test/next/host/reference-source.test.ts similarity index 71% rename from packages/chord/test/next/host/member-source.test.ts rename to packages/chord/test/next/host/reference-source.test.ts index 4a804cdb7..e4e235836 100644 --- a/packages/chord/test/next/host/member-source.test.ts +++ b/packages/chord/test/next/host/reference-source.test.ts @@ -1,12 +1,14 @@ -// Tests of member sources (slice 6a §1.2, §1.4): a local object's state members as references and other objects' -// derives follow them across attempts: passed on while live, frozen while failed, re-bound by name when live again. +// Tests of a reference's members (`ReferenceSource`; slice 6a §1.2, §1.4; decision "References are the only +// indirection"): consumers' watches and derives over `ref.member` pass straight through to the instance's member while +// live, keep the last value while it points at nothing (one `failed` call for a failure), move by name to the next +// instance (one frame, whole after a failure). import { describe, expect, it } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { arr, int, keyed, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { ChordError } from "../../../src/next/data/index.ts"; import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; -import { createHost, defineFacet, type Self, type StoreSelf } from "../../../src/next/host/index.ts"; +import { createHost, defineFacet, type Reference, type Self, type StoreSelf } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; import { resolveWatchable } from "../../../src/next/host/reference.ts"; import { constant } from "../../../src/next/state/index.ts"; @@ -16,12 +18,14 @@ import { delivered, expectCollected, failLive, + flushed, holdNextCommit, instancesOf, linked, loaded, memoryStore, metered, + Server, transported, valuesOf, } from "../support.ts"; @@ -95,7 +99,7 @@ describe("following the live attempt", () => { }); describe("a failed object's members", () => { - it("keep their last values and get no frames; a new watch opens at the last value (slice 6a §1.4)", async () => { + it("keep their last values and get the failure once; a new watch opens at the last value (slice 6a §1.4)", async () => { const { host, store, lists, fail } = await served(); await store.add({ text: "a" }, context); const counted = changesOf(await store.count.watch(context)); @@ -111,13 +115,15 @@ describe("a failed object's members", () => { store.retry(); await (await host.get(Store, {}, context)).add({ text: "b" }, context); await delivered(); - expect(valuesOf(counted)).toEqual([2]); - expect(valuesOf(listed)).toEqual([{ items: ["a", "b"] }]); - expect(valuesOf(lately)).toEqual([2]); + // The failure once; after it, whole values: the next instance's (unchanged) 1, then 2. + expect(valuesOf(counted)).toEqual([{ failed: "boom" }, 1, 2]); + // Opened while failed: at the last value, the failure once, then whole values. + expect(valuesOf(listed)).toEqual([{ failed: "boom" }, { items: ["a"] }, { items: ["a", "b"] }]); + expect(valuesOf(lately)).toEqual([{ failed: "boom" }, 1, 2]); await host.close(context); }); - it("re-exposed from another object freeze with the re-exposing object; the provider's source goes on", async () => { + it("re-exposed from another object hold with the re-exposing object; the provider's source goes on", async () => { const Note = obj({ title: str() }); const Memo = defineState("memo", { schema: Note, version: 1, initial: () => ({ title: "" }), keyed: true }); const Own = defineState("own", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); @@ -166,7 +172,7 @@ describe("a failed object's members", () => { await delivered(); expect(exposed.state.value.kind).toBe("failed"); expect([exposed.n.value, exposed.note.value]).toEqual([1, { title: "x" }]); - expect([exposedN, exposedNote]).toEqual([[], []]); + expect([exposedN, exposedNote].map((calls) => calls.map((call) => call.kind))).toEqual([["failed"], ["failed"]]); expect(provider.n.value).toBe(7); expect(valuesOf(providerN)).toEqual([7]); await host.close(context); @@ -174,7 +180,7 @@ describe("a failed object's members", () => { }); describe("the window of a live failure", () => { - it("members following another object's sources freeze at once, while an admitted commit of the object runs", async () => { + it("every member follows on while an admitted commit of the object runs, then all hold (audit A10)", async () => { const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); const Persisted = defineState("own", { schema: obj({ n: int() }), @@ -223,12 +229,14 @@ describe("the window of a live failure", () => { failLive(attempted, exposer!, new ChordError({ code: "unavailable", message: "boom" })); await provider.set(9, context); await delivered(); - expect([exposed.n.value, frames]).toEqual([1, []]); + // A re-exposed member shows one more change of its provider in the window. + expect([exposed.n.value, frames.map((call) => call.kind)]).toEqual([9, ["value"]]); held.release(); await change; await delivered(); - // The object's own admitted commit still publishes once. - expect(valuesOf(owned)).toEqual([{ n: 5 }]); + // The object's own admitted commit still publishes once; then every member holds (the failure once). + expect(valuesOf(owned)).toEqual([{ n: 5 }, { failed: "boom" }]); + expect([exposed.n.value, frames.map((call) => call.kind)]).toEqual([9, ["value", "failed"]]); await host.close(context); await store.close(context); }); @@ -291,8 +299,9 @@ describe("re-binding when a new attempt goes live", () => { await delivered(); expect(viewer.state.value.kind).toBe("live"); expect([store.count.value, viewer.tens.value]).toEqual([2, 20]); + // A derive over the held member keeps its last value (no failure); the member's watch gets it once. expect(valuesOf(tens)).toEqual([10, 20]); - expect(valuesOf(counted)).toEqual([1, 2]); + expect(valuesOf(counted)).toEqual([1, { failed: "boom" }, 1, 2]); await host.close(context); }); @@ -324,7 +333,12 @@ describe("re-binding when a new attempt goes live", () => { describe("subscribers", () => { it("a hold over a link on a remote object re-exposing a local member gets frames again after the local retry", async () => { const Count = local("app.count", { version: 1, params: {}, members: { n: obj({ n: int() }), add: method() } }); - const View = remote("app.view", { version: 1, params: {}, variants: { v: { state: { n: obj({ n: int() }) } } } }); + const View = remote("app.view", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: obj({ n: int() }) } } }, + }); const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); const selves: Self[] = []; const { server, client } = await linked((h) => { @@ -406,10 +420,11 @@ describe("a member the new attempt's variant lacks", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() } }, view: { state: { m: int() } } }, }); const owner = { kind: "owner" } as const; - const host = createHost({ principal: owner, onError: () => {} }); + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); const selves: Self[] = []; const facet = defineFacet((h) => h.serve(Two, (_, _principal, self) => { @@ -430,6 +445,8 @@ describe("a member the new attempt's variant lacks", () => { two.retry(); (await host.get(Two, {}, context)).release(); await delivered(); + // The failed instance is followed until its admitted commits published (audit A10), when the retry's attempt + // loads already (no failure call for it); then the new variant's lack of it. expect(calls).toEqual([ { kind: "failed", error: expect.objectContaining({ code: "invalid", reason: "member" }) }, ]); @@ -506,3 +523,125 @@ describe("member sources and memory", () => { await expectCollected(...refs); }); }); + +describe('what a reference keeps (decision "References are the only indirection")', () => { + it("keeps nothing of an ended instance while it waits with an open watch, its failure's error included", async () => { + const Count = local("app.kept", { version: 1, params: {}, members: { n: int() } }); + const Doc = defineState("kept", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 5 }) }); + const selves: WeakRef[] = []; + const facet = defineFacet((h) => + h.serve(Count, (_, self) => { + selves.push(new WeakRef(self)); + const doc = self.state(Doc); + return { n: self.derive([doc], ([value]) => value.n) }; + }), + ); + const host = createHost({ onError: () => {} }); + const from = attempted.length; + await host.load([{ name: "@app/kept", facet }], context); + let ref: Reference | undefined = await host.get(Count, {}, context); + const watch = await ref.n.watch(context); + const calls = changesOf(watch); + const objects = attempted.slice(from); + const documents = objects.map((object) => new WeakRef(object.deref()!.documents)); + // Removed: the reference waits with its last value and its failure; the instance ends. + await host.reload([], context); + await delivered(); + const state = ref.state.value; + if (state.kind !== "failed") throw new Error("expected failed"); + const kept = state.cause.error; + expect([ref.n.value, valuesOf(calls)]).toEqual([5, [{ failed: kept.message }]]); + await expectCollected(...selves, ...objects, ...documents); + // Released: its last values and bindings go with it. + const released = new WeakRef(ref); + await watch.stop(); + ref.release(); + ref = undefined; + await expectCollected(released); + expect(kept.code).toBe("unavailable"); + await host.close(context); + }); + + it("never takes values from an instance it never attached to: a reference made while its object is failed has none", async () => { + const { host, store, fail } = await served(); + await store.add({ text: "a" }, context); + fail(); + await delivered(); + const late = host.hold(Store, {}, context); + expect(store.count.value).toBe(1); + expect(() => late.count.value).toThrow("boom"); + late.release(); + await host.close(context); + }); + + it("computes a consumer's derive over a watched reference member once per commit (cost guard)", async () => { + const { telemetry, counts } = await metered(); + const { host, store } = await served({ telemetry }); + const viewer = await host.get(Viewer, {}, context); + const counted = changesOf(await store.count.watch(context)); + const tens = changesOf(await viewer.tens.watch(context)); + await store.add({ text: "a" }, context); + await delivered(); + const before = counts()["chord.derive.computes"] ?? 0; + for (const text of ["b", "c", "d"]) await store.add({ text }, context); + await delivered(); + // The store's count and the viewer's tens: one compute each per commit, nothing for passing through. + expect((counts()["chord.derive.computes"] ?? 0) - before).toBe(6); + expect([valuesOf(counted), valuesOf(tens)]).toEqual([ + [1, 2, 3, 4], + [10, 20, 30, 40], + ]); + await host.close(context); + }); +}); + +describe("a reference keeps the last values of its attached instance's variant (final review)", () => { + it("keeps an unread member of a remote contract served on its host when the type is removed (Astra)", async () => { + const Type = remote("app.remote-unread", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { full: { state: { n: int() } } }, + }); + const owner = { kind: "owner" } as const; + const facet = defineFacet((h) => + h.serve(Type, () => ({ variant: "full", members: { n: constant(7) } })), + ); + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); + await host.load([{ name: "@app/remote-unread", facet }], context); + const ref = await host.get(Type, {}, context); + // `n` never read: the removal takes its last value from the instance's variant (the object is live no more). + await host.reload([], context); + await delivered(); + expect(ref.n.value).toBe(7); + ref.release(); + await host.close(context); + }); +}); + +describe("a failed object lets its ended instance go (final review)", () => { + it("collects the instance while a reference keeps its last value with an open watch (Astra, GC)", async () => { + const Type = local("app.failed-gc", { version: 1, params: {}, members: { n: int() } }); + let self: Self | undefined; + const facet = defineFacet((h) => + h.serve(Type, (_, declared) => { + self = declared; + return { n: constant(7) }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/failed-gc", facet }], context); + const ref = await host.get(Type, {}, context); + changesOf(await ref.n.watch(context)); + const instance = new WeakRef(self!); + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + self = undefined; + await flushed(); + await delivered(); + // The object stays failed (host lifetime) and its reference keeps 7: neither keeps the ended instance. + expect(ref.n.value).toBe(7); + await expectCollected(instance); + ref.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/reference.test.ts b/packages/chord/test/next/host/reference.test.ts index adeeb7288..50b8b5372 100644 --- a/packages/chord/test/next/host/reference.test.ts +++ b/packages/chord/test/next/host/reference.test.ts @@ -13,6 +13,7 @@ import { type Self, } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; +import { Binding } from "../../../src/next/host/reference.ts"; import { RemoteObject } from "../../../src/next/remote/remote-object.ts"; import { constant } from "../../../src/next/state/index.ts"; import type { Store } from "../../../src/next/store/index.ts"; @@ -29,6 +30,7 @@ import { memoryStore, Notes, rejection, + Server, settledHeap, } from "../support.ts"; @@ -338,12 +340,13 @@ describe("remote references", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, }); const Holder = local("app.holder", { version: 1, params: {}, members: { n: int() } }); const seen: unknown[] = []; const owner = { kind: "owner" } as const; - const host = createHost({ principal: owner, onError: () => {} }); + const host = createHost({ address: Server(), principal: owner, onError: () => {} }); const facet = defineFacet((h) => { h.serve(Two, () => ({ variant: "view", members: { m: constant(1) } })); h.serve(Holder, (_, self) => { @@ -369,6 +372,7 @@ describe("remote reference types", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, }); type Ref = Reference; @@ -384,6 +388,7 @@ describe("remote reference types", () => { const Two = remote("app.two", { version: 1, params: {}, + hostAddress: () => Server(), variants: { full: { state: { n: int() }, methods: { bump: method() } }, view: { state: { m: int() } } }, }); expectTypeOf["variant"]>().toEqualTypeOf<"full" | "view" | undefined>(); @@ -473,20 +478,20 @@ describe("keyed state members (slice 6a §12)", () => { await host.close(context); }); - it("two consumers of one key share one source and one pin; each hold is released once, with its reference", async () => { + it("two consumers of one key each hold their own pin (references own their keys); each released once, with its reference", async () => { const { host, store, memos, pinned } = await notes(); await store.write({ id: "a", title: "x" }, context); const first = await host.get(NoteView, { id: "a", copy: 1 }, context); const second = await host.get(NoteView, { id: "a", copy: 2 }, context); const slot = (memos() as unknown as { slot: { source(key: string): { pins: number } | undefined } }).slot; const [one, two] = pinned.map((at) => (at as unknown as { source: object }).source); - // The store object's source of the key, which follows its attempts. - expect(one).toBe(two); + // Each reference's own source of the key, on the store's instance (decision "References are the only indirection"). + expect(one).not.toBe(two); const shared = slot.source("a")!; - expect(shared.pins).toBe(1); + expect(shared.pins).toBe(2); first.release(); await flushed(); - expect(slot.source("a")).toBe(shared); + expect([slot.source("a"), shared.pins]).toEqual([shared, 1]); second.release(); await flushed(); expect(slot.source("a")).toBeUndefined(); @@ -609,7 +614,7 @@ describe("keyed state members (slice 6a §12)", () => { await store.close(context); }); - it("a pin load that fails its provider fails the consumer as a dependency failure (transient)", async () => { + it("a pin load that fails its provider parks the consumer failed { waiting } on it (a dependency failure)", async () => { const store = await seeded(); const { host, errors } = await persistedNotes(store, persistedMemo(2)); const provider = await host.get(NoteStore, {}, context); @@ -617,7 +622,7 @@ describe("keyed state members (slice 6a §12)", () => { await vi.waitFor(() => expect(view.state.value.kind).toBe("failed")); expect(view.state.value).toMatchObject({ kind: "failed", - cause: { kind: "transient", error: { message: expect.stringContaining("cannot migrate") } }, + cause: { kind: "waiting", error: { message: expect.stringContaining("cannot migrate") } }, }); expect(provider.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); expect(errors.map(({ kind }) => kind)).toContain("object_failed"); @@ -654,3 +659,44 @@ describe("keyed state members (slice 6a §12)", () => { await host.close(context); }); }); + +describe("a binding released while it binds (fix round 5)", () => { + it("ends cleanly when its object's hold releases it at once, keeping nothing (Sol)", async () => { + const Type = local("app.reentrant", { version: 1, params: {}, members: { x: int() } }); + const facet = defineFacet((h) => h.serve(Type, () => ({ x: constant(1) }))); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/reentrant", facet }], context); + let ref = await host.get(Type, {}, context); + const released = new WeakRef(ref); + // The type is removed: the reference waits, with its last values. + await host.reload([], context); + const bind = Binding.prototype.bind; + const failures: unknown[] = []; + const binding = vi.spyOn(Binding.prototype, "bind").mockImplementation(function (this: Binding, target) { + try { + bind.call(this, target); + } catch (error) { + failures.push(error); + throw error; + } + }); + // `ChordObject.hold` lets a presence derive release a holder synchronously: the binding ends inside its bind. + const hold = ChordObject.prototype.hold; + const holding = vi.spyOn(ChordObject.prototype, "hold").mockImplementation(function (this: ChordObject, holder) { + hold.call(this, holder); + ref.release(); + }); + try { + await host.reload([{ name: "@app/reentrant", facet }], context); + expect(holding).toHaveBeenCalledOnce(); + } finally { + holding.mockRestore(); + binding.mockRestore(); + } + expect(failures).toEqual([]); + expect(ref.state.value.kind).toBe("ended"); + ref = undefined as never; + await expectCollected(released); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/retry-on.test.ts b/packages/chord/test/next/host/retry-on.test.ts new file mode 100644 index 000000000..31710c226 --- /dev/null +++ b/packages/chord/test/next/host/retry-on.test.ts @@ -0,0 +1,225 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; +import type { FacetHost, Self, StoreSelf } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { expectCollected, failLive, flushed, instancesOf, loaded } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** The models registry: the number of models contributed; it throws when told to. */ +const Models = defineRegistry("app.models", { version: 1 }); + +const Runtime = local("app.runtime", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Models` (throwing for a negative sum) and `Runtime`, contributing a writable count of models. */ +function serveModels(h: FacetHost): void { + h.serve(Models, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error("negative"); + return sum; + }, + }); + h.serve(Runtime, (_, self) => { + const count = self.writable(0); + self.contribute(Models, count); + return { + set: async (n: number) => { + count.set(n); + return null; + }, + }; + }); +} + +const User = local("app.user", { version: 1, params: { id: int() }, members: {} }); + +/** + * Serves `User`, whose open reads the models first and fails broken when there were none (after `wait`, if given); + * `opens` counts its attempts. + */ +function serveUser(h: FacetHost, opens: { n: number }, wait?: () => Promise): void { + h.serve(User, (_, self) => { + const models = self.registry(Models); + self.retryOn(models); + self.open( + async () => { + const seen = models.value; + opens.n++; + await wait?.(); + if (seen === 0) throw broken(new ChordError({ code: "unavailable", message: "no model" })); + }, + () => {}, + ); + return {}; + }); +} + +describe("retryOn", () => { + it("retries a failed object (a broken open) when a source gets a new value: the watches were installed before the open", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + expect(user.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + await (await host.get(Runtime, {}, context)).set(1, context); + await flushed(); + expect(user.state.value).toEqual({ kind: "live" }); + expect(opens.n).toBe(2); + await host.close(context); + }); + + it("does nothing while the object is parked failed { waiting }: its dependencies decide (dependency model §3.8)", async () => { + const Missing = local("app.missing", { version: 1, params: {}, members: {} }); + let factories = 0; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(User, (_, self) => { + factories++; + self.retryOn(self.registry(Models)); + self.get(Missing, {}); + return {}; + }); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + expect(user.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await (await host.get(Runtime, {}, context)).set(1, context); + await flushed(); + expect([user.state.value.kind, factories]).toEqual(["failed", 1]); + await host.close(context); + }); + + it("wakes on value calls only: a throwing compose wakes nothing, its next good composition does", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + const runtime = await host.get(Runtime, {}, context); + await runtime.set(-1, context); + await flushed(); + expect(opens.n).toBe(1); + await runtime.set(2, context); + await flushed(); + expect(opens.n).toBe(2); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("remembers a wake while it activates: that attempt failing is retried at once", async () => { + const opens = { n: 0 }; + const gate = Promise.withResolvers(); + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens, async () => { + if (opens.n === 1) await gate.promise; + }); + }); + const runtime = await host.get(Runtime, {}, context); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + // The first attempt read no model and waits; a model arrives meanwhile; then that attempt fails broken. + await runtime.set(1, context); + expect(user.state.value).toEqual({ kind: "loading" }); + gate.resolve(); + await flushed(); + expect(opens.n).toBe(2); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("ignores wakes while live", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(User, (_, self) => { + self.retryOn(self.registry(Models)); + self.open( + async () => void opens.n++, + () => {}, + ); + return {}; + }); + }); + const user = host.hold(User, { id: 1 }, context); + await flushed(); + await (await host.get(Runtime, {}, context)).set(3, context); + await flushed(); + expect(opens.n).toBe(1); + expect(user.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); + + it("refuses sources that end with an attempt: a reference's member, its own writable, `ref.state`", async () => { + const Other = local("app.other", { version: 1, params: {}, members: { n: int() } }); + const Picky = local("app.picky", { version: 1, params: { which: int() }, members: {} }); + const { host } = await loaded((h) => { + h.serve(Other, () => ({ n: constant(1) })); + h.serve(Picky, ({ which }, self) => { + const other = self.get(Other, {}); + const sources = [other.n, self.writable(1), other.state, constant(1)]; + self.retryOn(sources[which]!); + return {}; + }); + }); + for (const which of [0, 1, 2]) { + await expect(host.get(Picky, { which }, context)).rejects.toThrow( + `app.picky(${which}): self.retryOn takes registry values and constants (sources that outlive the attempt)`, + ); + } + (await host.get(Picky, { which: 3 }, context)).release(); + await host.close(context); + }); + + it("are the object's: a failed object that ends releases them, and is collected while the registry lives", async () => { + const opens = { n: 0 }; + const { host } = await loaded((h) => { + serveModels(h); + serveUser(h, opens); + }); + const refs = await (async () => { + const user = host.hold(User, { id: 1 }, context); + await flushed(); + user.release(); + await flushed(); + return objects.filter((ref) => ref.deref()?.label === "app.user(1)"); + })(); + expect(refs.length).toBeGreaterThan(0); + await expectCollected(...refs); + await host.close(context); + }); + + it("does not wake an object by its own contributions leaving (it failed live: no retry without backoff)", async () => { + const Contributor = local("app.contributor", { version: 1, params: {}, members: {} }); + let contributor: StoreSelf | undefined; + const { host } = await loaded((h) => { + serveModels(h); + h.serve(Contributor, (_, self) => { + contributor = self; + self.contribute(Models, 1); + self.retryOn(self.registry(Models)); + return {}; + }); + }); + const models = await host.registry(Models, context); + const watched: number[] = []; + (await models.watch(context)).start(async (call) => void (call.kind === "value" && watched.push(call.value))); + failLive(objects, contributor as Self, new ChordError({ code: "unavailable", message: "boom" })); + await flushed(); + const reference = host.hold(Contributor, {}, context); + await flushed(); + expect(reference.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); + expect(watched).toEqual([0]); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/revocation.test.ts b/packages/chord/test/next/host/revocation.test.ts index a80fba200..a73b222f8 100644 --- a/packages/chord/test/next/host/revocation.test.ts +++ b/packages/chord/test/next/host/revocation.test.ts @@ -1,12 +1,12 @@ // Tests of revocation (§4, R10, R11): revocation keys, \`host.revoke\` on one host (objects, running calls, activations, -// holds still waiting for \`load\`), child links and \`revoke\` messages, GC, and revocation across a relay. +// holds bound or not), child links and \`revoke\` messages, GC, and revocation across a relay. import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; -import { int, local, remote } from "../../../src/next/contracts/index.ts"; +import { hostKind, int, local, remote } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet, type FacetHost, type Host } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; -import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; import { jsonPart } from "../../../src/next/link/internal.ts"; import { constant } from "../../../src/next/state/index.ts"; import { @@ -18,7 +18,10 @@ import { RawPeer, rawParent, rawWorkers, + rejection, relayed, + SERVER_ADDRESS, + Server, Session, serveNotes, type TestPrincipal, @@ -50,9 +53,10 @@ const Barrier = local("app.barrier", { version: 1, params: {}, members: { n: int /** A guest's variant: its `wait` method's index. */ const { wait } = wireNames(Notes, "reader").methods as { wait: number }; -/** A worker of session S1 serving `Notes` (as `serveNotes`), its parent a raw peer whose holds name their principal. */ +/** A worker of session S1 serving `Notes` (as `serveNotes`, at the address `Notes` names), its parent a raw peer whose holds name their principal. */ async function worker(setup: (host: FacetHost) => void = (h) => void serveNotes(h)) { const host = createHost({ + address: Server(), principal: { kind: "worker", session: "S1" }, revokedBy, onError: () => {}, @@ -71,6 +75,7 @@ function holdAs(peer: RawPeer, hold: number, principal: TestPrincipal, Type: typ version: Type.version, fingerprint: Type.fingerprint, params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, principal, }); } @@ -79,6 +84,7 @@ describe("revocation keys", () => { it("are computed once per principal object, compared as canonical JSON", async () => { let calls = 0; const host = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: () => { calls++; @@ -103,6 +109,7 @@ describe("revocation keys", () => { it("keep no principal alive (a WeakMap keyed by the principal the host cached them for)", async () => { const cached: WeakRef[] = []; const host = createHost({ + address: Server(), principal: { kind: "worker", session: "S1" }, revokedBy: (principal) => { // The principal decoded on this host, the cache's key: not the sender's object. @@ -172,7 +179,12 @@ forEachCodec(() => { }); it("abandons an activation in progress (race 1): its open's context is aborted, its holders get gone", async () => { - const Slow = remote("app.slow", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Slow = remote("app.slow", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const aborted: unknown[] = []; const { host, parent } = await worker((h) => h.serve(Slow, (_, _principal, self) => { @@ -196,6 +208,7 @@ forEachCodec(() => { version: Slow.version, fingerprint: Slow.fingerprint, params: "{}", + address: SERVER_ADDRESS, principal: S1, }); await transported(); @@ -217,6 +230,7 @@ forEachCodec(() => { // A worker connects before it loads (W6); a load can wait for an activation (the start barrier). const opened = Promise.withResolvers(); const host = createHost({ + address: Server(), principal: { kind: "worker", session: "S1" }, revokedBy, onError: () => {}, @@ -299,6 +313,22 @@ forEachCodec(() => { await host.close(context); }); + it("ends an unbound hold of the principal (refused, waiting for a provider): gone, nothing left of it", async () => { + const { host, parent } = await worker(); + const Absent = { ...Notes, name: "app.absent" } as unknown as typeof Notes; + holdAs(parent, 0, S1, Absent); + expect(await parent.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(host.activity.value.references).toBe(1); + parent.send({ kind: "revoke", revocation: { session: "S1" } }); + expect(await parent.next("gone")).toMatchObject({ + hold: 0, + error: { kind: "chord", error: { code: "forbidden", reason: "revoked" } }, + }); + await transported(); + expect(host.activity.value.references).toBe(0); + await host.close(context); + }); + it("binds a hold made for a revoked principal, then the parent's revoke after it ends it (race 3, R10's window)", async () => { const { host, parent } = await worker(); holdAs(parent, 0, S1); @@ -313,6 +343,7 @@ forEachCodec(() => { describe("links and revoke messages", () => { it("closes the child links whose principal it revokes, never the parent link", async () => { const host = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy, onError: () => {}, @@ -336,6 +367,7 @@ forEachCodec(() => { it("ignores a revoke from a child link (race 6)", async () => { const host = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy, onError: () => {}, @@ -428,3 +460,127 @@ describe("revocation across a relay", () => { expect(again.state.value.kind).toBe("live"); }); }); + +describe("revocation of this host's own requests over its paths (fix round 1)", () => { + type Guest = { kind: "owner" } | { kind: "guest"; grant: string }; + const There = hostKind("app-there"); + const Remote = remote("app.remote", { + version: 1, + params: {}, + hostAddress: () => There(), + variants: { view: { state: {}, methods: {} } }, + }); + const guestRevokedBy = (p: Guest) => (p.kind === "guest" ? [p.grant] : []); + + it("ends a guest's ctx.get waiting for its route before any link exists, at once (Astra 1, Sol 1)", async () => { + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + host.route(There, { authorize: () => true, resolve: () => new Promise(() => {}) }, context); + await host.load([], context); + let outcome: unknown; + void rejection( + host.root((ctx) => ctx.get(Remote, {}, { readyTimeout: 60_000 }), context, { + principal: { kind: "guest", grant: "g" }, + }), + ).then((error) => { + outcome = error; + }); + await transported(); + host.revoke("g", context); + await transported(); + expect(outcome).toMatchObject({ code: "forbidden", reason: "revoked" }); + await host.close(context); + }); + + it("never lets the revoked request bind on the route's later link (Astra 1)", async () => { + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + const gate = Promise.withResolvers<{ link: Link; principal: Guest; trusted: boolean }>(); + host.route(There, { authorize: () => true, resolve: () => gate.promise }, context); + await host.load([], context); + const pending = rejection( + host.root( + async (ctx) => { + await ctx.get(Remote, {}, { readyTimeout: 500 }); + throw new Error("revoked request became live"); + }, + context, + { principal: { kind: "guest", grant: "g" } }, + ), + ); + await transported(); + host.revoke("g", context); + const worker = createHost({ + address: There(), + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + const [up, down] = memoryLinkPair(); + worker.connect(up, { principal: { kind: "owner" }, trusted: true }, context); + const facet = defineFacet((h) => h.serve(Remote, () => ({ variant: "view", members: {} }))); + await worker.load([{ name: "@test/late", facet }], context); + gate.resolve({ link: down, principal: { kind: "owner" }, trusted: true }); + expect(await pending).toMatchObject({ code: "forbidden", reason: "revoked" }); + await transported(); + expect(worker.activity.value.references).toBe(0); + await host.close(context); + await worker.close(context); + }); + + it("ends a guest's reference over a route link that dropped, and nothing holds it on the next one", async () => { + const routed: RawPeer[] = []; + // The drop resolves again at once (slice 10 R2); the next link comes only after the revocation. + let next: () => void = () => {}; + const revoked = new Promise((resolve) => { + next = resolve; + }); + const host = createHost({ + principal: { kind: "owner" }, + revokedBy: guestRevokedBy, + onError: () => {}, + }); + host.route( + There, + { + authorize: () => true, + resolve: async () => { + if (routed.length > 0) await revoked; + const [a, b] = memoryLinkPair(); + routed.push(new RawPeer(b)); + return { link: a, principal: { kind: "owner" }, codec: testCodec() }; + }, + }, + context, + ); + await host.load([], context); + const states: string[] = []; + void host.root( + async (ctx) => { + const held = await ctx.get(Remote, {}, { readyTimeout: 60_000 }).catch((error: Error) => error); + states.push(held instanceof Error ? held.message : "live"); + }, + context, + { principal: { kind: "guest", grant: "g" } }, + ); + await transported(); + const first = routed[0]!; + await first.next("hold"); + first.close(); + await transported(); + host.revoke("g", context); + next(); + await transported(); + expect(states).toEqual(["app.remote: revoked"]); + expect(routed.flatMap((peer, i) => (i === 0 ? [] : peer.received.filter(({ kind }) => kind === "hold")))).toEqual( + [], + ); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/root.test.ts b/packages/chord/test/next/host/root.test.ts new file mode 100644 index 000000000..0847d0477 --- /dev/null +++ b/packages/chord/test/next/host/root.test.ts @@ -0,0 +1,208 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { type Ctx, createHost, defineFacet, sleep } from "../../../src/next/host/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { expectCollected, flushed, loaded, rejection, type TestPrincipal } from "../support.ts"; + +const Info = local("app.info", { version: 1, params: { id: int() }, members: { id: int(), wait: method() } }); + +/** Serves `Info`; its `wait` waits until the caller's context aborts. */ +function serveInfo(h: Parameters[0]>[0]): void { + h.serve(Info, ({ id }) => ({ + id: constant(id), + wait: async (ctx) => { + await sleep(60_000, ctx); + return null; + }, + })); +} + +describe("host.root", () => { + it("gives fn a call context: the host's principal (or the given one), this host's incarnation, `ctx.get`", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([{ name: "@app/info", facet: defineFacet(serveInfo) }], context); + const seen = await host.root(async (ctx) => { + const info = await ctx.get(Info, { id: 7 }); + return { principal: ctx.principal, incarnation: ctx.incarnation, id: info.id.value }; + }, context); + expect(seen).toMatchObject({ principal: { kind: "server" }, id: 7 }); + const owner = { kind: "owner", access: "local" } as const; + expect(await host.root(async (ctx) => ctx.principal, context, { principal: owner })).toEqual(owner); + await host.close(context); + }); + + it("runs anonymous with an explicit undefined principal; only an omitted one takes the host's (slice 10 §6)", async () => { + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([], context); + expect(await host.root(async (ctx) => ctx.principal, context, { principal: undefined })).toBeUndefined(); + expect(await host.root(async (ctx) => ctx.principal, context, {})).toEqual({ kind: "server" }); + await host.close(context); + }); + + // Opus's build review F5: an anonymous scope's `ctx.get` of a remote type served here failed on a WeakMap key. + it("refuses an anonymous scope a remote type served here (forbidden); a local one it gets", async () => { + const Server = hostKind("server"); + const Secrets = remote("app.secrets", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { methods: { read: method(undefined, str()) } } }, + }); + const host = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const facet = defineFacet((h) => { + serveInfo(h); + h.serve(Secrets, () => ({ variant: "main", members: { read: async () => "secret" } })); + }); + await host.load([{ name: "@app/secrets", facet }], context); + const anonymous = { principal: undefined }; + const refused = await rejection(host.root(async (ctx) => ctx.get(Secrets, {}), context, anonymous)); + expect(refused).toMatchObject({ + code: "forbidden", + message: "app.secrets: an anonymous caller gets no remote object", + }); + expect(await host.root(async (ctx) => (await ctx.get(Info, { id: 3 })).id.value, context, anonymous)).toBe(3); + await host.close(context); + }); + + it("seals the context when fn settles: its references released, its signal aborted, `ctx.get` refused `released`", async () => { + const { host } = await loaded(serveInfo); + let kept: Ctx | undefined; + const info = await host.root(async (ctx) => { + kept = ctx; + return ctx.get(Info, { id: 1 }); + }, context); + expect(info.state.value).toMatchObject({ kind: "ended", error: { code: "released" } }); + expect(kept!.abortSignal?.aborted).toBe(true); + expect(await rejection(kept!.get(Info, { id: 1 }))).toMatchObject({ code: "released", reason: "settled" }); + // A throwing fn seals too, and rejects with its error. + await expect( + host.root(async () => { + throw new Error("fn failed"); + }, context), + ).rejects.toThrow("fn failed"); + await host.close(context); + }); + + it("an abort rejects at once and seals: a running call through its reference rejects, a later get is `released`", async () => { + const { host } = await loaded(serveInfo); + const { context: cancellable, cancel } = withCancel(context); + let kept: Ctx | undefined; + let call: Promise | undefined; + const running = host.root(async (ctx) => { + kept = ctx; + const info = await ctx.get(Info, { id: 1 }); + call = info.wait(context); + // Uncooperative: it waits on nothing of Chord's. + await new Promise(() => {}); + }, cancellable); + await flushed(); + cancel(new Error("stop")); + expect(await rejection(running)).toEqual(new Error("stop")); + expect(await rejection(call!)).toMatchObject({ code: "released" }); + expect(await rejection(kept!.get(Info, { id: 2 }))).toMatchObject({ code: "released", reason: "settled" }); + await host.close(context); + }); + + it("an aborted context rejects before fn runs; a synchronous throw is a rejection", async () => { + const { host } = await loaded(serveInfo); + const { context: aborted, cancel } = withCancel(context); + cancel(new Error("gone")); + let ran = false; + const fn = async (): Promise => { + ran = true; + }; + await expect(host.root(fn, aborted)).rejects.toThrow("gone"); + expect(ran).toBe(false); + await expect( + host.root( + (() => { + throw new Error("sync"); + }) as never, + context, + ), + ).rejects.toThrow("sync"); + await host.close(context); + }); + + it("a `ctx.get` before `load` resolved parks until it did", async () => { + const host = createHost({ onError: () => {} }); + const result = host.root(async (ctx) => (await ctx.get(Info, { id: 3 })).id.value, context); + await flushed(); + await host.load([{ name: "@app/info", facet: defineFacet(serveInfo) }], context); + expect(await result).toBe(3); + await host.close(context); + }); + + it("`host.close` seals every open scope (an fn sleeping on its ctx ends); after it, `root` rejects `released`", async () => { + const { host } = await loaded(serveInfo); + const sleeping = rejection(host.root((ctx) => sleep(60_000, ctx), context)); + await flushed(); + await host.close(context); + expect(await sleeping).toMatchObject({ code: "released", reason: "closed" }); + expect(await rejection(host.root(async () => 1, context))).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("aborts the scope's own signal when fn settles and at `host.close`, the caller's context having one (Fable, Opus, Sol)", async () => { + const { host } = await loaded(serveInfo); + const settled = withCancel(context); + let kept: Ctx | undefined; + await host.root(async (ctx) => { + kept = ctx; + }, settled.context); + expect(kept!.abortSignal?.aborted).toBe(true); + expect(settled.context.abortSignal?.aborted).toBe(false); + const closing = withCancel(context); + let slept: "running" | "ended" = "running"; + const sleeping = host.root(async (ctx) => { + try { + await sleep(60_000, ctx); + } finally { + slept = "ended"; + } + }, closing.context); + const outcome = rejection(sleeping); + await flushed(); + await host.close(context); + await flushed(); + expect(slept).toBe("ended"); + expect(closing.context.abortSignal?.aborted).toBe(false); + expect(await outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("`host.close` rejects the caller at once, whatever an uncooperative fn still does (Sol)", async () => { + const { host } = await loaded(serveInfo); + const running = host.root(() => new Promise(() => {}), context); + const outcome = running.then( + () => "resolved", + (error: unknown) => error, + ); + await host.close(context); + expect(await outcome).toMatchObject({ code: "released", reason: "closed" }); + }); + + it("leaves no listener on the caller's signal once `host.close` rejected an fn that never settles (Astra)", async () => { + const controller = new AbortController(); + const host = createHost({ onError: () => {} }); + const running = host.root(async () => new Promise(() => {}), withAbortSignal(controller.signal, context)); + const rejected = running.catch((error: unknown) => error); + await host.close(context); + expect(await rejected).toMatchObject({ code: "released", reason: "closed" }); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(0); + }); + + it("keeps nothing once sealed: its scope and references are collected while the caller's signal lives on", async () => { + const { host } = await loaded(serveInfo); + const { context: long } = withCancel(context); + const refs = await (async () => { + const made: WeakRef[] = []; + await host.root(async (ctx) => { + made.push(new WeakRef(ctx), new WeakRef(await ctx.get(Info, { id: 1 }))); + }, long); + return made; + })(); + await expectCollected(...refs); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/route.test.ts b/packages/chord/test/next/host/route.test.ts index eb3d6413f..3d9fbf1f8 100644 --- a/packages/chord/test/next/host/route.test.ts +++ b/packages/chord/test/next/host/route.test.ts @@ -12,6 +12,7 @@ import { PlacedNotes, rejection, relayed, + Server, Session, serveNotes, type TestPrincipal, @@ -101,56 +102,88 @@ forEachCodec(() => { }); }); - it("fails a hold broken invalid/placement for an address whose params do not fit the kind", async () => { + it("fails a hold broken invalid/host_address for an address whose params do not fit the kind", async () => { const Misplaced = remote("app.misplaced", { version: 1, params: { n: int() }, - // An address of the right kind with params of the wrong type, as a buggy placement could compute. - placement: ({ n }) => ({ kind: "session", params: { id: n }, key: "" }) as never, + // An address of the right kind with params of the wrong type, as a buggy `hostAddress` could compute. + hostAddress: ({ n }) => ({ kind: "session", params: { id: n }, key: "" }) as never, variants: { v: { state: { n: int() } } }, }); const { tab, resolves } = await relayed(); expect(await rejection(tab.get(Misplaced, { n: 1 }, context))).toMatchObject({ code: "invalid", - reason: "placement", + reason: "host_address", }); expect(resolves).toEqual([]); }); }); - describe("resolve failures and re-resolves", () => { + describe("resolve failures and re-resolves (no timer, slice 10 R2)", () => { fakeTimers(); - it("a rejected resolve leaves the holds loading, is reported, and resolves again after the address's backoff (race 2)", async () => { - let failures = 2; - const { tab, resolves, errors } = await relayed({ + it("a rejected resolve fails its holders waiting with its error, is reported, and runs no timer (race 2)", async () => { + let attempts = 0; + const { tab, server, errors } = await relayed({ + resolve: async () => { + attempts++; + throw new Error("no worker"); + }, + }); + // One relayed from the tab, one the server's own. + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + const own = server.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + for (const ref of [notes, own]) { + expect(ref.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "session|S1: no worker" } }, + }); + } + expect(errors.server).toMatchObject([{ kind: "resolve_failed", address: "session|S1" }]); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("a new hold resolves a failed address again; the earlier holders bind on its link without a re-hold (Q1)", async () => { + let fail = true; + const { tab, resolves } = await relayed({ resolve: async (_address, worker) => { - if (failures-- > 0) throw new Error("no worker yet"); + if (fail) throw new Error("no worker"); + return worker(); + }, + }); + const first = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(first.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + fail = false; + const second = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(resolves).toEqual(["session|S1"]); + expect([first.state.value, second.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); + + it("RouteHandle.retry resolves a failed address now; a retry of an address nobody holds keeps nothing", async () => { + let fail = true; + const { tab, route, resolves } = await relayed({ + resolve: async (_address, worker) => { + if (fail) throw new Error("no worker"); return worker(); }, }); const notes = tab.hold(PlacedNotes, { session: "S1" }, context); await transported(); - expect(notes.state.value).toEqual({ kind: "loading" }); - expect(errors.server).toMatchObject([{ kind: "resolve_failed", address: "session|S1" }]); - await elapse(999); - expect(errors.server).toHaveLength(1); - await elapse(1); - // The second failure doubles the window. - expect(errors.server).toHaveLength(2); - await elapse(1999); - expect(notes.state.value).toEqual({ kind: "loading" }); - await elapse(1); - expect(resolves).toEqual(["session|S1"]); + fail = false; + route.retry(Session({ id: "S1" })); + route.retry(Session({ id: "S9" })); + await transported(); expect(notes.state.value).toEqual({ kind: "live" }); + expect(resolves).toEqual(["session|S1"]); }); - it("a returned link that is closed already starts the backoff", async () => { - let closed = true; - const { tab, resolves } = await relayed({ - resolve: async (address, worker) => { - if (!closed) return worker(); - closed = false; + it("a returned link closed already fails the holders", async () => { + const { tab } = await relayed({ + resolve: async (address) => { const [a, b] = memoryLinkPair(); b.close(); await flushed(); @@ -159,20 +192,35 @@ forEachCodec(() => { }); const notes = tab.hold(PlacedNotes, { session: "S1" }, context); await transported(); - expect(notes.state.value).toEqual({ kind: "loading" }); - await elapse(1000); - expect(resolves).toEqual(["session|S1"]); - expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "session|S1: the link closed before its handshake" } }, + }); }); - it("a dropped link resolves again while held: references go loading, then live on a fresh worker (race 6)", async () => { + it("a link closing before its handshake fails its holders: no re-resolve loop over a worker dying at start", async () => { + let attempts = 0; + const { tab } = await relayed({ + resolve: async (address) => { + attempts++; + const [a, b] = memoryLinkPair(); + setTimeout(() => b.close(), 10); + return { link: a, principal: { kind: "worker", session: address.params.id } }; + }, + }); + const notes = tab.hold(PlacedNotes, { session: "S1" }, context); + await elapse(10); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await elapse(120_000); + expect(attempts).toBe(1); + }); + + it("a dropped link that completed its handshake resolves again at once, once (race 6)", async () => { const { tab, resolves, routeLinks, workers } = await relayed(); const notes = await tab.get(PlacedNotes, { session: "S1" }, context); const first = workers.get("session|S1")!; routeLinks[0]!.close(); await transported(); - expect(notes.state.value).toEqual({ kind: "loading" }); - await elapse(1000); expect(resolves).toEqual(["session|S1", "session|S1"]); expect(workers.get("session|S1")).not.toBe(first); expect(notes.state.value).toEqual({ kind: "live" }); @@ -181,42 +229,13 @@ forEachCodec(() => { expect(notes.notes.value).toEqual([{ text: "fresh" }]); }); - it("only a link that completed its handshake resets the backoff", async () => { - let silent = 0; - const { tab, resolves, routeLinks } = await relayed({ - resolve: async (address, worker) => { - if (silent-- <= 0) return worker(); - // A link whose other end never greets and closes. - const [a, b] = memoryLinkPair(); - setTimeout(() => b.close(), 10); - return { link: a, principal: { kind: "worker", session: address.params.id } }; - }, - }); - const notes = await tab.get(PlacedNotes, { session: "S1" }, context); - silent = 1; - routeLinks[0]!.close(); - await elapse(1000); - // The silent link closed before its handshake: the window doubles. - await elapse(10); - await elapse(1999); - expect(resolves).toEqual(["session|S1"]); - await elapse(1); - expect(resolves).toEqual(["session|S1", "session|S1"]); - expect(notes.state.value).toEqual({ kind: "live" }); - // That link greeted: the next drop starts again at 1 s. - routeLinks.at(-1)!.close(); - await elapse(1000); - expect(resolves).toHaveLength(3); - }); - - it("a link whose peer never greets closes after 30 s of silence; the address backs off, then resolves again while held", async () => { + it("a link whose peer never greets closes after 30 s of silence and fails its holders", async () => { // The heartbeat's timer too, before the hosts start it. vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "setInterval", "clearInterval", "Date"], now: 0 }); - let silent = true; - const { tab, resolves } = await relayed({ - resolve: async (address, worker) => { - if (!silent) return worker(); - silent = false; + let attempts = 0; + const { tab } = await relayed({ + resolve: async (address) => { + attempts++; // Its other end is never read: no hello, no close. const [a] = memoryLinkPair(); return { link: a, principal: { kind: "worker", session: address.params.id } }; @@ -224,14 +243,12 @@ forEachCodec(() => { }); const notes = tab.hold(PlacedNotes, { session: "S1" }, context); await elapse(29_000); - expect(resolves).toEqual([]); expect(notes.state.value).toEqual({ kind: "loading" }); - // The heartbeat checks every 1.5 s: the silent link closes within it, then its address waits 1 s. + // The heartbeat checks every 1.5 s: the silent link closes within it. await elapse(2_500); - await elapse(1_000); - expect(resolves).toEqual(["session|S1"]); - await transported(); - expect(notes.state.value).toEqual({ kind: "live" }); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await elapse(120_000); + expect(attempts).toBe(1); }); it("an address without holds is not resolved again after its link drops", async () => { @@ -275,6 +292,42 @@ forEachCodec(() => { }); }); + describe("RouteHandle.suspend (R1, Q3)", () => { + it("resolves nothing while held: the holders fail waiting suspended once the link is gone; retry lifts it", async () => { + const { tab, route, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + route.suspend(Session({ id: "S1" })); + expect(notes.state.value).toEqual({ kind: "live" }); + routeLinks[0]!.close(); + await transported(); + expect(notes.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "session|S1: suspended" } }, + }); + const another = tab.hold(PlacedNotes, { session: "S1" }, context); + await transported(); + expect(another.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + expect(resolves).toEqual(["session|S1"]); + route.retry(Session({ id: "S1" })); + await transported(); + expect(resolves).toEqual(["session|S1", "session|S1"]); + expect([notes.state.value, another.state.value]).toEqual([{ kind: "live" }, { kind: "live" }]); + }); + + it("goes with the last holder: a later hold resolves on demand; an unheld address is never suspended", async () => { + const { tab, route, resolves, routeLinks } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + route.suspend(Session({ id: "S1" })); + routeLinks[0]!.close(); + await transported(); + notes.release(); + await transported(); + route.suspend(Session({ id: "S1" })); + await tab.get(PlacedNotes, { session: "S1" }, context); + expect(resolves).toEqual(["session|S1", "session|S1"]); + }); + }); + describe("host.close during a resolve (race 7)", () => { it("aborts the resolve's context and closes a link returned afterwards", async () => { let routing: AbortSignal | undefined; @@ -322,11 +375,16 @@ forEachCodec(() => { const Who = remote("app.who", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { app: method(undefined, str()) } } }, }); const resolved: (string | undefined)[] = []; let worker: Host | undefined; - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); server.route( Session, { @@ -369,6 +427,37 @@ forEachCodec(() => { }); describe("GC", () => { + it("a failed or suspended address keeps no route state once its holders are released", async () => { + const { refs, server } = await (async () => { + const refs: WeakRef[] = []; + const { tab, server, route, routeLinks, workers, errors } = await relayed({ + authorize: (_principal, address) => { + refs.push(new WeakRef(address)); + return true; + }, + resolve: async (address, worker) => { + if (address.params.id === "failed") throw new Error("no worker"); + return worker(); + }, + }); + const failed = tab.hold(PlacedNotes, { session: "failed" }, context); + const held = await tab.get(PlacedNotes, { session: "held" }, context); + route.suspend(Session({ id: "held" })); + routeLinks[0]!.close(); + await transported(); + for (const ref of [failed, held]) ref.release(); + await transported(); + // The reported error's stack keeps the test's `resolve` frame (and its address) alive, the worker host its + // own address: not the route's. + errors.server.length = 0; + routeLinks.length = 0; + workers.clear(); + return { refs, server }; + })(); + await expectCollected(...refs); + expect(server).toBeDefined(); + }); + it("a closed route link and its released references keep no route state", async () => { const { refs, tab } = await (async () => { const { tab, routeLinks, workers } = await relayed(); diff --git a/packages/chord/test/next/host/routines.test.ts b/packages/chord/test/next/host/routines.test.ts index e3bb69a6e..7dfa40565 100644 --- a/packages/chord/test/next/host/routines.test.ts +++ b/packages/chord/test/next/host/routines.test.ts @@ -36,6 +36,7 @@ import { loaded, Notes, rawParent, + Server, serveNotes, type TestPrincipal, transported, @@ -726,8 +727,9 @@ describe("on", () => { }, ctx); }); }); + // 0 → 1 → 2 → 3, then the write of 3 changes nothing and nothing calls again (waited for, not timed: Opus F3). + await vi.waitFor(() => expect(calls).toBe(4), { timeout: 2_000 }); await new Promise((resolve) => setTimeout(resolve, 20)); - // 0 → 1 → 2 → 3, then the write of 3 changes nothing and nothing calls again. expect(calls).toBe(4); }); @@ -789,9 +791,10 @@ describe("on", () => { const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, }); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const serve = defineFacet((h) => h.serve(Halves, (_, _principal, self) => { const doc = self.state(Doc); @@ -844,7 +847,7 @@ describe("on", () => { }); it("follows ref.state: a reference over a link going loading and live again", async () => { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const Viewer = local("app.viewer", { version: 1, params: { id: int() }, members: {} }); const states: string[] = []; @@ -1069,7 +1072,12 @@ describe("routines and the end", () => { }); it("act as a remote object's principal", async () => { - const Watcher = remote("app.watcher", { version: 1, params: {}, variants: { v: { state: {} } } }); + const Watcher = remote("app.watcher", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: {} } }, + }); const seen: unknown[] = []; const { client } = await linked( (h) => diff --git a/packages/chord/test/next/host/staging.test.ts b/packages/chord/test/next/host/staging.test.ts new file mode 100644 index 000000000..b0b01b563 --- /dev/null +++ b/packages/chord/test/next/host/staging.test.ts @@ -0,0 +1,516 @@ +// Tests of staging (slice 7b §2, per plugin since slice 7c): a reload's setups and recorded factories; each kind of +// throw keeps the failing plugin on its old code, reported; recorded declarations decide an object's fate. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { + createHost, + defineFacet, + type FacetHost, + type Host, + type PluginFacet, + type PluginFailure, + type StoreSelf, +} from "../../../src/next/host/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; +import { flushed, memoryStore, Server } from "../support.ts"; + +const Store = local("app.store", { version: 1, params: {}, members: { label: str(), bump: method(undefined, int()) } }); + +/** Serves `Store` labelled `label`, its factory running `declare` first. */ +function serveStore(label: string, declare: (self: StoreSelf) => void = () => {}) { + return defineFacet((h: FacetHost) => + h.serve(Store, (_, self) => { + declare(self); + return { label: constant(label), bump: async () => 0 }; + }), + ); +} + +/** The error of the one plugin `facets`' reload reports failed (it kept its old code). */ +async function failedReload(host: Host, facets: readonly PluginFacet[]): Promise { + const { failed } = await host.reload(facets, context); + expect(failed).toHaveLength(1); + return failed[0]!.error as Error; +} + +describe("staging: a throw keeps the plugin on its old code, reported", () => { + const throwing: [string, ReturnType, string][] = [ + [ + "a setup", + defineFacet(() => { + throw new Error("boom"); + }), + 'plugin "@test/app": boom', + ], + [ + "a factory", + serveStore("v2", () => { + throw new Error("boom"); + }), + 'plugin "@test/app", app.store: boom', + ], + [ + "a declaration error", + serveStore("v2", (self) => { + self.state(defineState("d", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })); + self.state(defineState("d", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })); + }), + 'state "d" is declared twice', + ], + [ + "a document at an older version than the object's", + serveStore("v2", (self) => + self.state(defineState("doc", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })), + ), + "version 1 is older than its documents' (2)", + ], + [ + "a broken accepts table", + defineFacet((h) => + h.serve(Store, () => ({ label: constant("x"), bump: async () => 0 }), { + accepts: { 7: {} } as never, + }), + ), + "accepts[7]: not an older version", + ], + ]; + for (const [what, facet, message] of throwing) { + it(`${what}: the reload reports it, the plugin unchanged`, async () => { + const host = createHost({ onError: () => {} }); + const v2 = defineState("doc", { + schema: obj({ n: int() }), + version: 2, + initial: () => ({ n: 0 }), + migrate: () => ({ n: 0 }), + }); + await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => self.state(v2)) }], context); + const store = await host.get(Store, {}, context); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain(message); + expect(store.label.value).toBe("v1"); + expect(store.state.value).toEqual({ kind: "live" }); + // The plugin is still the old one: reloading it with the same facet set again is a change of nothing. + await host.close(context); + }); + } + + it("a remote type on a host without an address is refused, as at load", async () => { + const Remote = remote("app.remote", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); + const host = createHost({ principal: { kind: "server" }, onError: () => {} }); + await host.load([], context); + const facet = defineFacet((h) => h.serve(Remote, () => ({ variant: "v", members: { n: constant(1) } }))); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain( + "a host without an address serves only local types", + ); + await host.close(context); + }); +}); + +describe("staging: recorded declarations decide the fate", () => { + it("records without activating: no native opens, no reference binds, no document is made", async () => { + const Dep = local("app.dep", { version: 1, params: {}, members: { n: int() } }); + const opened: string[] = []; + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: serveStore("v1") }], context); + const store = await host.get(Store, {}, context); + const facet = serveStore("v2", (self) => { + self.open( + async () => void opened.push("opened"), + () => {}, + ); + self.get(Dep, {}); + self.state(defineState("fresh", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) })); + }); + const report = await host.reload([{ name: "@test/app", facet }], context); + // It opens natives now: a restart, whose activation opened its native once. + expect(report.objects).toEqual([{ object: "app.store", kind: "restarted" }]); + await flushed(); + // Neither staging nor the restart opened it: the new attempt parks on the missing `Dep` first. + expect(opened).toEqual([]); + expect(store.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + }); + + it("a table of the new factory at a newer version than the store's restarts the object (its eager migration)", async () => { + const store = await memoryStore(); + const V1 = defineTable("items", { version: 1, columns: { id: str(), name: str() }, key: ["id"] }); + const V2 = evolveTable(V1, (t) => t.rename("name", "title")); + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => self.table(V1)) }], context); + await host.get(Store, {}, context); + const same = await host.reload( + [{ name: "@test/app", facet: serveStore("v2", (self) => self.table(V1)) }], + context, + ); + expect(same.objects).toEqual([{ object: "app.store", kind: "swapped" }]); + const newer = await host.reload( + [{ name: "@test/app", facet: serveStore("v3", (self) => self.table(V2)) }], + context, + ); + expect(newer.objects).toEqual([{ object: "app.store", kind: "restarted" }]); + await host.close(context); + await store.close(context); + }); + + it("a job declared at its version with another schema is refused at staging", async () => { + const Job1 = defineJob("app.job", { version: 1, args: obj({}), result: int() }); + const Job1b = defineJob("app.job", { version: 1, args: obj({ x: int() }), result: int() }); + const host = createHost({ onError: () => {} }); + const run = { run: async () => 1 }; + await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => void self.job(Job1, run)) }], context); + await host.get(Store, {}, context); + const facet = serveStore("v2", (self) => void self.job(Job1b, run)); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain( + "schema changed without a version bump", + ); + await host.close(context); + }); + + it("a contribution outside a host-lifetime object is a declaration error at staging", async () => { + const Names = defineRegistry("app.names", { version: 1 }); + const Page = local("app.page", { version: 1, params: { id: int() }, members: { id: int() } }); + const host = createHost({ onError: () => {} }); + const page = (contribute: boolean) => + defineFacet((h) => + h.serve(Page, ({ id }, self) => { + if (contribute) (self as unknown as StoreSelf).contribute(Names, "x"); + return { id: constant(id) }; + }), + ); + await host.load([{ name: "@test/app", facet: page(false) }], context); + await host.get(Page, { id: 1 }, context); + expect((await failedReload(host, [{ name: "@test/app", facet: page(true) }])).message).toContain( + "self.contribute is valid only in a local type with empty params", + ); + await host.close(context); + }); +}); + +describe("staging: declaration parity with an attempt (fix round 1)", () => { + it("refuses retryOn of an attempt-owned writable, the plugin kept on its old code", async () => { + const T = local("astra.staging", { version: 1, params: {}, members: { n: int() } }); + const oldFacet = defineFacet((h) => h.serve(T, () => ({ n: constant(1) }))); + const newFacet = defineFacet((h) => + h.serve(T, (_, self) => { + self.retryOn(self.writable(0)); + return { n: constant(2) }; + }), + ); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@astra/staging", facet: oldFacet }], context); + const old = await host.get(T, {}, context); + const { failed } = await host.reload([{ name: "@astra/staging", facet: newFacet }], context); + const state = old.state.value; + old.release(); + await host.close(context); + expect({ failed: failed.map(({ plugin }) => plugin), state: state.kind }).toEqual({ + failed: ["@astra/staging"], + state: "live", + }); + }); +}); + +describe("staging per plugin (slice 7c §4)", () => { + const NotesV1 = local("app.notes", { version: 1, params: {}, members: { count: int() } }); + const NotesV2 = local("app.notes", { version: 2, params: {}, members: { count: int(), title: str() } }); + const View = local("app.view", { version: 1, params: {}, members: { label: str() } }); + const Extra = local("app.extra", { version: 1, params: {}, members: { n: int() } }); + + /** The notes provider at `version`, labelled through its count. */ + const notes = (version: 1 | 2, count: number) => + defineFacet((h) => + version === 1 + ? h.serve(NotesV1, () => ({ count: constant(count) })) + : h.serve(NotesV2, () => ({ count: constant(count), title: constant("t") })), + ); + /** A consumer of the notes at `version`, labelled `label` and the count it reads. */ + const view = (version: 1 | 2, label: string) => + defineFacet((h) => + h.serve(View, (_, self) => { + const count = (version === 1 ? self.get(NotesV1, {}) : self.get(NotesV2, {})).count; + return { label: self.derive([count], ([n]) => `${label} ${n}`) }; + }), + ); + const throwing = defineFacet(() => { + throw new Error("setup threw"); + }); + const failedOf = (report: { readonly failed: readonly PluginFailure[] }) => + report.failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + + it("a failing plugin keeps its old code; the others of the reload apply", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/store", facet: serveStore("v1") }, + ], + context, + ); + const store = await host.get(Store, {}, context); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: throwing }, + { name: "@test/store", facet: serveStore("v2") }, + ], + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/store"], removed: [] }); + expect(failedOf(report)).toEqual([["@test/notes", 'plugin "@test/notes": setup threw']]); + expect(report.objects).toEqual([{ object: "app.store", kind: "swapped" }]); + await flushed(); + expect([store.label.value, counted.count.value, counted.state.value]).toEqual(["v2", 1, { kind: "live" }]); + // Fixed later, it applies as any change does. + const fixed = await host.reload( + [ + { name: "@test/notes", facet: notes(1, 2) }, + { name: "@test/store", facet: serveStore("v2") }, + ], + context, + ); + expect([fixed.plugins.changed, fixed.failed]).toEqual([["@test/notes", "@test/store"], []]); + await host.close(context); + }); + + it("a factory throwing in staging fails its plugin alone; an added plugin that fails is left out", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/store", facet: serveStore("v1") }, + ], + context, + ); + await host.get(Store, {}, context); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/extra", facet: throwing }, + { name: "@test/notes", facet: notes(1, 2) }, + { + name: "@test/store", + facet: serveStore("v2", () => { + throw new Error("factory threw"); + }), + }, + ], + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/notes"], removed: [] }); + expect(failedOf(report)).toEqual([ + ["@test/extra", 'plugin "@test/extra": setup threw'], + ["@test/store", 'plugin "@test/store", app.store: factory threw'], + ]); + await flushed(); + expect(counted.count.value).toBe(2); + await host.close(context); + }); + + it("the closure rule: a consumer edited together with a failing provider keeps its old code", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: throwing }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect(report.plugins.changed).toEqual([]); + expect(failedOf(report)).toEqual([ + ["@test/notes", 'plugin "@test/notes": setup threw'], + ["@test/view", 'plugin "@test/view": needs app.notes v2, which plugin "@test/notes" serves on its old code'], + ]); + expect(report.objects).toEqual([]); + await flushed(); + expect([shown.label.value, shown.state.value]).toEqual(["old 1", { kind: "live" }]); + // Both fixed together: both apply. + const both = await host.reload( + [ + { name: "@test/notes", facet: notes(2, 5) }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect([both.plugins.changed, both.failed]).toEqual([["@test/notes", "@test/view"], []]); + await flushed(); + expect(shown.label.value).toBe("new 5"); + await host.close(context); + }); + + it("takes the plugins whose new code failed before staging (`failed`): the closure rule and the report see them", async () => { + const host = createHost({ onError: () => {} }); + const provider = notes(1, 1); + await host.load( + [ + { name: "@test/notes", facet: provider }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + // Its new build failed: the caller passes its loaded facet, and says so. + const before = [{ plugin: "@test/notes", error: new Error("build failed") }]; + const report = await host.reload( + [ + { name: "@test/notes", facet: provider }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + { failed: before }, + ); + expect(failedOf(report)).toEqual([ + ["@test/notes", "build failed"], + ["@test/view", 'plugin "@test/view": needs app.notes v2, which plugin "@test/notes" serves on its old code'], + ]); + expect([report.plugins.changed, shown.label.value]).toEqual([[], "old 1"]); + await host.close(context); + }); + + it("the closure rule checks a request as a real bind does, the kept provider's host address included (Astra)", async () => { + const Here = hostKind("app.here"); + const Elsewhere = hostKind("app.elsewhere"); + const placed = (only: boolean) => + remote("app.provider", { + version: 1, + params: { id: int() }, + hostAddress: ({ id }) => (only && id !== 1 ? Elsewhere() : Here()), + variants: { main: { state: { n: int() } } }, + }); + const Consumer = local("app.consumer", { version: 1, params: {}, members: { n: int() } }); + const consumer = (id: number) => + defineFacet((h) => + h.serve(Consumer, (_, self) => ({ n: self.derive([self.get(placed(false), { id }).n], ([n]) => n) })), + ); + const host = createHost({ address: Here(), principal: { kind: "owner" }, onError: () => {} }); + await host.load( + [ + // The provider's own code places every id but 1 on another host. + { + name: "@test/provider", + facet: defineFacet((h) => + h.serve(placed(true), ({ id }) => ({ variant: "main", members: { n: constant(id) } })), + ), + }, + { name: "@test/consumer", facet: consumer(1) }, + ], + context, + ); + const shown = await host.get(Consumer, {}, context); + const report = await host.reload( + [ + { name: "@test/provider", facet: throwing }, + { name: "@test/consumer", facet: consumer(2) }, + ], + context, + ); + expect(report.failed.map(({ plugin }) => plugin)).toEqual(["@test/provider", "@test/consumer"]); + expect([shown.state.value, shown.n.value]).toEqual([{ kind: "live" }, 1]); + await host.close(context); + }); + + it("the closure rule covers a name nobody serves while a plugin failed, and runs to a fixpoint", async () => { + const Middle = local("app.middle", { version: 2, params: {}, members: { n: int() } }); + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const host = createHost({ onError: () => {} }); + const middle = (needs: boolean) => + defineFacet((h) => h.serve(Middle, (_, self) => ({ n: needs ? self.get(Extra, {}).n : constant(0) }))); + const top = (needs: boolean) => + defineFacet((h) => h.serve(Top, (_, self) => ({ n: needs ? self.get(Middle, {}).n : constant(0) }))); + await host.load( + [ + { name: "@test/middle", facet: middle(false) }, + { name: "@test/top", facet: top(false) }, + ], + context, + ); + await host.get(Top, {}, context); + await host.get(Middle, {}, context); + // The new extra plugin fails: the middle's new code needs Extra, served by nobody now; the top's new code needs + // the middle, which keeps its old code (another version is not involved: its request binds to the old one). + const report = await host.reload( + [ + { name: "@test/extra", facet: throwing }, + { name: "@test/middle", facet: middle(true) }, + { name: "@test/top", facet: top(true) }, + ], + context, + ); + expect(failedOf(report)).toEqual([ + ["@test/extra", 'plugin "@test/extra": setup threw'], + ["@test/middle", 'plugin "@test/middle": needs app.extra v1, which no plugin serves now (a plugin failed)'], + ]); + expect(report.plugins.changed).toEqual(["@test/top"]); + await host.close(context); + }); + + it("without a failing plugin, a request that does not bind is an ordinary wait: the consumer applies and parks", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect([report.plugins.changed, report.failed]).toEqual([["@test/notes", "@test/view"], []]); + expect(report.objects).toEqual([ + { object: "app.notes", kind: "swapped" }, + expect.objectContaining({ object: "app.view", kind: "parked" }), + ]); + await flushed(); + expect(shown.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + }); + + it("a type moving between plugins while its old owner fails: the new owner collides and keeps its old code too", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/a", facet: notes(1, 1) }, + { name: "@test/b", facet: defineFacet(() => {}) }, + ], + context, + ); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/a", facet: throwing }, + { name: "@test/b", facet: notes(1, 2) }, + ], + context, + ); + expect(failedOf(report)).toEqual([ + ["@test/a", 'plugin "@test/a": setup threw'], + ["@test/b", 'plugin "@test/b": app.notes is served twice (plugin "@test/a")'], + ]); + expect(counted.count.value).toBe(1); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/host/unbound.test.ts b/packages/chord/test/next/host/unbound.test.ts new file mode 100644 index 000000000..c5430a447 --- /dev/null +++ b/packages/chord/test/next/host/unbound.test.ts @@ -0,0 +1,207 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, str } from "../../../src/next/contracts/index.ts"; +import { ChordError, type Principal } from "../../../src/next/data/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { Binding } from "../../../src/next/host/reference.ts"; +import { UnboundIndex, type UnboundRequest } from "../../../src/next/host/unbound.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { jsonPart } from "../../../src/next/link/internal.ts"; +import { Hold } from "../../../src/next/remote/hold.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { + delivered, + expectCollected, + instancesOf, + RawPeer, + rejection, + SERVER_ADDRESS, + Server, + type TestPrincipal, + testCodec, + transported, +} from "../support.ts"; + +const bindings = instancesOf(Binding, "release"); +const holds = instancesOf(Hold, "release"); + +const Info = local("app.info", { version: 1, params: {}, members: { title: str() } }); +const Missing = local("app.missing", { version: 1, params: { id: int() }, members: { n: int() } }); + +/** A request recording its wakes and ends. */ +function request(principal?: Principal): UnboundRequest & { readonly log: string[] } { + const log: string[] = []; + return { principal, log, wake: () => log.push("wake"), end: (error) => log.push(`end ${error.message}`) }; +} + +describe("the index", () => { + it("wakes every request at the publication, out of the index first; a deleted one is not woken", () => { + const index = new UnboundIndex(); + const [a, b, c] = [request(), request(), request()]; + index.add("app.a", a); + index.add("app.a", b); + index.add("app.b", c); + index.delete("app.a", b); + index.wakeAll(); + expect([a.log, b.log, c.log]).toEqual([["wake"], [], ["wake"]]); + // Out of it: a second publication wakes nobody. + index.wakeAll(); + expect(a.log).toEqual(["wake"]); + }); + + it("wakes the requests of the names a reload serves only (carry-over: publications wake per name)", () => { + const index = new UnboundIndex(); + const [a, b, c] = [request(), request(), request()]; + index.add("app.a", a); + index.add("app.b", b); + index.add("app.c", c); + index.wake(["app.a", "app.c", "app.unknown"]); + expect([a.log, b.log, c.log]).toEqual([["wake"], [], ["wake"]]); + // The others stay indexed until their own names are served. + index.wake(["app.b"]); + expect(b.log).toEqual(["wake"]); + // Woken out of the index: a later publication wakes none of them again. + index.wakeAll(); + expect([a.log, b.log, c.log]).toEqual([["wake"], ["wake"], ["wake"]]); + }); + + it("ends the requests of a revoked principal only, and every request at the host's close", () => { + const index = new UnboundIndex(); + const guest = request({ kind: "guest" }); + const owner = request({ kind: "owner" }); + const own = request(); + for (const each of [guest, owner, own]) index.add("app.a", each); + const revoked = new ChordError({ code: "forbidden", reason: "revoked", message: "revoked" }); + index.revoke((principal) => principal.kind === "guest", revoked); + expect([guest.log, owner.log, own.log]).toEqual([["end revoked"], [], []]); + index.close(new ChordError({ code: "released", reason: "closed", message: "closed" })); + expect([owner.log, own.log]).toEqual([["end closed"], ["end closed"]]); + }); +}); + +describe("references waiting in the index", () => { + it("before the publication: loading, placed by it; a type not served then waits failed { waiting }", async () => { + const host = createHost({ onError: () => {} }); + const info = host.hold(Info, {}, context); + const missing = host.hold(Missing, { id: 1 }, context); + expect([info.state.value, missing.state.value]).toEqual([{ kind: "loading" }, { kind: "loading" }]); + await host.load( + [{ name: "@test/a", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + await delivered(); + expect(info.state.value).toEqual({ kind: "live" }); + expect(missing.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable", message: "app.missing is not served here" } }, + retryAt: undefined, + }); + // A retry leaves it waiting: an event wakes it, nothing else. + missing.retry(); + expect(missing.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + expect(missing.state.value).toMatchObject({ kind: "ended", error: { reason: "closed" } }); + }); + + it("leave nothing behind once released, before and after the publication (GC)", async () => { + const host = createHost({ onError: () => {} }); + host.hold(Missing, { id: 1 }, context).release(); + await host.load([], context); + host.hold(Missing, { id: 2 }, context).release(); + const got = host.get(Missing, { id: 3 }, context, { readyTimeout: 1 }).catch(() => {}); + await got; + await expectCollected(...bindings); + expect(host).toBeDefined(); + }); + it("end at the start of the host's close, not after its slow native closes (fix round 1, Sol 6)", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + const closeGate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + const facet = defineFacet((h) => + h.serve(Core, (_, self) => { + self.open( + async () => 1, + () => closeGate.promise, + ); + return { n: constant(1) }; + }), + ); + await host.load([{ name: "@test/a", facet }], context); + let outcome: unknown; + void rejection(host.get(Missing, { id: 1 }, context)).then((error) => { + outcome = error; + }); + const held = host.hold(Missing, { id: 2 }, context); + const closing = host.close(context); + await delivered(); + expect(outcome).toMatchObject({ code: "released", reason: "closed" }); + expect(held.state.value).toMatchObject({ kind: "ended", error: { reason: "closed" } }); + closeGate.resolve(); + await closing; + }); +}); + +describe("holds waiting in the index", () => { + /** A server not loaded yet, a raw consumer peer on a child link, and a hold of `app.missing` it sent. */ + async function early() { + const host = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const [a, b] = memoryLinkPair(); + host.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + peer.send({ + kind: "hold", + hold: 0, + type: "app.missing", + version: 1, + fingerprint: "0123456789abcdef", + params: jsonPart({}), + address: SERVER_ADDRESS, + }); + await transported(); + return { host, peer, link: a }; + } + + it("wait for the publication without a word, then are refused failed { waiting } and kept", async () => { + const { host, peer } = await early(); + expect(peer.kinds()).toEqual(["hello"]); + await host.load([], context); + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(host.activity.value.references).toBe(1); + await host.close(context); + }); + + it("leave nothing behind once their link closed, before and after the publication (GC)", async () => { + const before = await early(); + before.link.close(); + await transported(); + await before.host.load([], context); + const after = await early(); + await after.host.load([], context); + await after.peer.next("failed"); + after.link.close(); + await transported(); + expect([before.host.activity.value.references, after.host.activity.value.references]).toEqual([0, 0]); + await expectCollected(...holds); + expect([before.host, after.host]).toBeDefined(); + }); +}); + +describe("what waits unserved (slice 12 §6, testPlugin)", () => { + it("lists the index's names sorted, and the host the names no plugin serves", async () => { + const index = new UnboundIndex(); + index.add("b", request()); + index.add("a", request()); + expect(index.names()).toEqual(["a", "b"]); + const host = createHost({ onError: () => {} }); + await host.load( + [{ name: "info", facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))) }], + context, + ); + const held = host.hold(Missing, { id: 1 }, context); + await delivered(); + expect((host as unknown as ChordHost).unservedNames()).toEqual(["app.missing"]); + held.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/job.test.ts b/packages/chord/test/next/jobs/job.test.ts index 4fd9aadcb..d82d82b3f 100644 --- a/packages/chord/test/next/jobs/job.test.ts +++ b/packages/chord/test/next/jobs/job.test.ts @@ -34,6 +34,7 @@ import { loaded, metered, rejection, + Server, type TestPrincipal, transported, } from "../support.ts"; @@ -1057,11 +1058,12 @@ describe("keep-alive", () => { const View = remote("app.view", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { running: arr(unknown()) } } }, }); let job: JobHandle<{ text: string }, string, undefined> | undefined; let ran = false; - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); const facet = defineFacet((h) => h.serve(View, (_, _principal, self) => { job = self.job(Echo, { @@ -1157,13 +1159,19 @@ describe("keep-alive", () => { describe("object end", () => { type Revocation = { session: string }; - const Jobs = remote("app.jobs", { version: 1, params: {}, variants: { v: { state: {} } } }); + const Jobs = remote("app.jobs", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: {} } }, + }); it("revocation (R11): runs' signals fire with the end's error, the drain waits for them, no finish", async () => { let job: JobHandle<{ text: string }, string, undefined> | undefined; const finished: string[] = []; let reason: unknown; const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), onError: () => {}, @@ -1209,6 +1217,7 @@ describe("object end", () => { const gate = Promise.withResolvers(); const events: string[] = []; const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: (p) => (p.kind === "guest" ? [{ session: p.session }] : []), onError: () => {}, diff --git a/packages/chord/test/next/jobs/persisted.test.ts b/packages/chord/test/next/jobs/persisted.test.ts index 80751d207..b144a2981 100644 --- a/packages/chord/test/next/jobs/persisted.test.ts +++ b/packages/chord/test/next/jobs/persisted.test.ts @@ -1064,3 +1064,36 @@ describe("GC", () => { await hanging.close(); }); }); + +describe("versions resolved by a reload (slice 7b §8)", () => { + it("a persisted run blocked for its version at its load is resumed by the reload that brings its version", async () => { + const open = await storeFile(); + const jobs = await opened(await open(), () => new Promise(() => {})); + const run = await jobs.imports.start({ path: "a" }, context); + await delivered(); + await jobs.crash(); + const store = await open(); + const finished: ImportOutcome[] = []; + const serve = (declared: typeof Import) => + defineFacet((h) => + h.serve(Store, (_, self) => { + self.job(declared, { + run: async () => ({ count: 7 }), + finish: async (outcome) => void finished.push(outcome), + }); + return {}; + }), + ); + const host = createHost({ store, onError: () => {} }); + await host.load([{ name: "notes", facet: serve(ImportV2) }], context); + await delivered(); + expect(finished).toEqual([]); + expect(await record(store, run.id)).toMatchObject({ version: 1, phase: "running" }); + await host.reload([{ name: "notes", facet: serve(Import) }], context); + await sleepMs(20); + expect(finished).toMatchObject([{ id: run.id, kind: "done", result: { count: 7 } }]); + expect(await record(store, run.id)).toBeUndefined(); + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/records.test.ts b/packages/chord/test/next/jobs/records.test.ts index d8d2fca8b..a6a0e0e2b 100644 --- a/packages/chord/test/next/jobs/records.test.ts +++ b/packages/chord/test/next/jobs/records.test.ts @@ -2,12 +2,13 @@ // seal, finished on the next live attempt, and the keep-alive of runs waiting for one. import { describe, expect, it } from "vitest"; -import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; -import { int, obj, str } from "../../../src/next/contracts/index.ts"; +import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { arr, int, local, method, obj, str, unknown } from "../../../src/next/contracts/index.ts"; import { defineState, type SingletonHandle } from "../../../src/next/documents/index.ts"; -import { untilAborted } from "../../../src/next/host/index.ts"; +import { createHost, defineFacet, untilAborted } from "../../../src/next/host/index.ts"; import { ChordObject } from "../../../src/next/host/object.ts"; import { defineJob, type JobHandle, type Outcome, type RunningEntry } from "../../../src/next/jobs/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; import type { StagedWrite } from "../../../src/next/store/step.ts"; import { delivered, flushed, instancesOf, memoryStore, rejection, restartable, storeRestartable } from "../support.ts"; @@ -203,3 +204,238 @@ describe("the object's end during an admitted commit", () => { await store.close(context); }); }); + +/** The jobs object of the reload tests: `start` starts a run of `text`, `running` lists them. */ +const Runner = local("app.runner", { + version: 1, + params: {}, + members: { start: method(obj({ text: str() }), str()), running: arr(unknown()) }, +}); + +/** Version 2 of `Work`: its args gained a priority, its result is an object; v1 runs map forwards. */ +const WorkV2 = defineJob("app.work", { + version: 2, + args: obj({ text: str(), priority: int() }), + result: obj({ text: str() }), + state: Progress, + accepts: { + 1: { args: (old: { text: string }) => ({ ...old, priority: 0 }), result: (old: string) => ({ text: old }) }, + }, +}); + +/** `WorkV2` without an adapter for version 1. */ +const WorkV2Strict = defineJob("app.work", { + version: 2, + args: obj({ text: str(), priority: int() }), + result: obj({ text: str() }), + state: Progress, +}); + +type Finished = { readonly code: string; readonly outcome: unknown }; + +/** + * A facet serving `Runner` with `job` (labelled `code` in what its finish records into `finished`); its `run` waits for + * `gates.shift()` when one is queued; with `native` the object opens a native (a reload restarts it). + */ +function runner( + code: string, + job: typeof Work | typeof WorkV2 | typeof WorkV2Strict | undefined, + finished: Finished[], + options: { readonly gates?: Promise[]; readonly native?: boolean } = {}, +) { + return defineFacet((h) => + h.serve(Runner, (_, self) => { + if (options.native) + self.open( + async () => code, + () => {}, + ); + if (job === undefined) return { start: async () => "", running: constant([]) }; + const handle = ( + self.job as (definition: unknown, implementation: unknown) => JobHandle + )(job, { + run: async (args: { text: string }, ctx: { readonly abortSignal?: AbortSignal }) => { + const gate = options.gates?.shift(); + if (gate !== undefined) await Promise.race([gate, untilAborted(ctx as never)]); + ctx.abortSignal?.throwIfAborted(); + return job.version === 1 ? `${args.text} by ${code}` : { text: `${args.text} by ${code}` }; + }, + finish: (outcome: unknown) => void finished.push({ code, outcome }), + }); + return { + start: async ({ text }: { text: string }, ctx: Context) => + (await handle.start((job.version === 1 ? { text } : { text, priority: 1 }) as never, ctx)).id, + running: handle.running as never, + }; + }), + ); +} + +describe("jobs across a reload (slice 7b §8)", () => { + it("an older run finishes on its old code across a swap; its finish runs on the new code, mapped forwards once", async () => { + const finished: Finished[] = []; + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise] }) }], context); + const jobs = await host.get(Runner, {}, context); + await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", WorkV2, finished) }], context); + await flushed(); + // Listed with its args mapped forwards. + expect(jobs.running.value).toMatchObject([{ version: 1, args: { text: "a", priority: 0 }, phase: "running" }]); + gate.resolve(); + await flushed(); + expect(finished).toEqual([ + { + code: "v2", + outcome: expect.objectContaining({ + kind: "done", + args: { text: "a", priority: 0 }, + result: { text: "a by v1" }, + }), + }, + ]); + await host.close(context); + }); + + it("a run without an adapter is blocked (version) until a reload brings matching code", async () => { + const finished: Finished[] = []; + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise] }) }], context); + const jobs = await host.get(Runner, {}, context); + const id = await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", WorkV2Strict, finished) }], context); + gate.resolve(); + await flushed(); + expect(jobs.running.value).toEqual([{ id, version: 1, phase: "blocked", reason: "version" }]); + expect(finished).toEqual([]); + await host.reload([{ name: "@test/app", facet: runner("v1b", Work, finished) }], context); + await flushed(); + expect(finished).toEqual([ + { code: "v1b", outcome: expect.objectContaining({ kind: "done", result: "a by v1" }) }, + ]); + expect(jobs.running.value).toEqual([]); + await host.close(context); + }); + + it("a native restart aborts running in-memory runs (reload); their finish runs on the new code", async () => { + const finished: Finished[] = []; + const host = createHost({ onError: () => {} }); + const gate = Promise.withResolvers(); + await host.load( + [{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise], native: true }) }], + context, + ); + const jobs = await host.get(Runner, {}, context); + await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", Work, finished, { native: true }) }], context); + await flushed(); + expect(finished).toEqual([ + { + code: "v2", + outcome: expect.objectContaining({ kind: "failed", error: { kind: "aborted", reason: "reload" } }), + }, + ]); + await host.close(context); + }); + + it("a job the new code no longer declares blocks its runs (version); declared again, they go on", async () => { + const finished: Finished[] = []; + const gate = Promise.withResolvers(); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/app", facet: runner("v1", Work, finished, { gates: [gate.promise] }) }], context); + const jobs = await host.get(Runner, {}, context); + await jobs.start({ text: "a" }, context); + await flushed(); + await host.reload([{ name: "@test/app", facet: runner("v2", undefined, finished) }], context); + gate.resolve(); + await flushed(); + expect(finished).toEqual([]); + await host.reload([{ name: "@test/app", facet: runner("v3", Work, finished) }], context); + await flushed(); + expect(finished).toEqual([{ code: "v3", outcome: expect.objectContaining({ kind: "done", result: "a by v1" }) }]); + await host.close(context); + }); +}); + +describe("a reservation released after a swap (final review)", () => { + it("starts the old code's run with its own args, not the new version's mapping (Sol)", async () => { + const Owner = local("app.reserve-owner", { version: 1, params: {}, members: {} }); + const V1 = defineJob("app.reserve", { + version: 1, + args: obj({ n: int() }), + result: int(), + persist: { resume: "rerun" }, + }); + const V2 = defineJob("app.reserve", { + version: 2, + args: obj({ label: str() }), + result: int(), + persist: { resume: "rerun" }, + accepts: { 1: { args: (old: { n: number }) => ({ label: `mapped ${old.n}` }), result: (old: number) => old } }, + }); + const store = await memoryStore(); + const host = createHost({ store, onError: () => {} }); + const seen: unknown[] = []; + let job: JobHandle<{ n: number }, number, undefined> | undefined; + await host.load( + [ + { + name: "@app/reserve", + facet: defineFacet((h) => + h.serve(Owner, (_, self) => { + job = self.job(V1, { + run: async (args) => { + seen.push(args.n); + return 42; + }, + }); + return {}; + }), + ), + }, + ], + context, + ); + // The run's reservation (the second commit) waits on `gate` until the swap to V2 happened. + const entered = Promise.withResolvers(); + const gate = Promise.withResolvers(); + const commit = store.commit.bind(store); + let writes = 0; + store.commit = (staged, fence, ctx) => { + if (++writes !== 2) return commit(staged, fence, ctx); + const held: StagedWrite = { + order: "write", + apply: async () => { + entered.resolve(); + await gate.promise; + return []; + }, + }; + return commit([...staged, held], fence, ctx); + }; + const starting = job!.start({ n: 1 }, context); + await entered.promise; + const run = await starting; + const v2 = defineFacet((h) => + h.serve(Owner, (_, self) => { + self.job(V2, { run: async () => 43 }); + return {}; + }), + ); + const reloading = host.reload([{ name: "@app/reserve", facet: v2 }], context); + await delivered(); + gate.resolve(); + await reloading; + await run.outcome(context); + // V1's handler got V1's args. + expect(seen).toEqual([1]); + store.commit = commit; + await host.close(context); + await store.close(context); + }); +}); diff --git a/packages/chord/test/next/jobs/run.test.ts b/packages/chord/test/next/jobs/run.test.ts index 1c2bb89a1..b430e62bc 100644 --- a/packages/chord/test/next/jobs/run.test.ts +++ b/packages/chord/test/next/jobs/run.test.ts @@ -8,8 +8,8 @@ import { rejection } from "../support.ts"; describe("a run", () => { it("has an entry per phase: args and state while running or finishing, neither when blocked, with its reason", () => { - const run = new JobRun("r1", 2, "k", { command: "ls" }, "running"); - expect(run.entryFor({ output: "" })).toEqual({ + const run = new JobRun("r1", 2, "f2", "k", { command: "ls" }, "running"); + expect(run.entryFor({ output: "" }, run.args)).toEqual({ id: "r1", version: 2, key: "k", @@ -18,7 +18,7 @@ describe("a run", () => { phase: "running", }); run.phase = "finishing"; - expect(new JobRun("r2", 1, undefined, { a: 1 }, "running").entryFor(undefined)).toEqual({ + expect(new JobRun("r2", 1, "f1", undefined, { a: 1 }, "running").entryFor(undefined, { a: 1 })).toEqual({ id: "r2", version: 1, args: { a: 1 }, @@ -26,23 +26,26 @@ describe("a run", () => { }); run.phase = "blocked"; run.blocked = "finish_failed"; - expect(run.entryFor({ output: "x" })).toEqual({ + expect(run.entryFor({ output: "x" }, run.args)).toEqual({ id: "r1", version: 2, phase: "blocked", reason: "finish_failed", }); run.blocked = "version"; - expect(run.entryFor(undefined)).toMatchObject({ phase: "blocked", reason: "version" }); + expect(run.entryFor(undefined, run.args)).toMatchObject({ phase: "blocked", reason: "version" }); // A persisted run waiting for its reservation, or stopped, runs (again): `running`. run.phase = "pending"; - expect(run.entryFor(undefined)).toMatchObject({ phase: "running" }); + expect(run.entryFor(undefined, run.args)).toMatchObject({ phase: "running" }); run.phase = "stopped"; - expect(run.entryFor(undefined)).toMatchObject({ phase: "running" }); + expect(run.entryFor(undefined, run.args)).toMatchObject({ phase: "running" }); + // An older run's entry shows its args as the current code takes them (mapped by its job). + run.phase = "running"; + expect(run.entryFor(undefined, { command: "ls -l" })).toMatchObject({ args: { command: "ls -l" } }); }); it("gives every waiter the outcome, late ones too; a waiter's context ends only its wait", async () => { - const run = new JobRun("r1", 1, undefined, {}, "running"); + const run = new JobRun("r1", 1, "f1", undefined, {}, "running"); const { context: waiting, cancel } = withCancel(context); const cancelled = rejection(run.handle.outcome(waiting)); cancel(new Error("left")); @@ -50,7 +53,7 @@ describe("a run", () => { const outcome = { id: "r1", args: {}, state: undefined, kind: "done", result: 1 } as const; run.resolve(outcome); expect(await run.handle.outcome(context)).toBe(outcome); - const rejected = new JobRun("r2", 1, undefined, {}, "running"); + const rejected = new JobRun("r2", 1, "f1", undefined, {}, "running"); rejected.reject(new ChordError({ code: "aborted", message: "dropped" })); expect(await rejection(rejected.handle.outcome(context))).toMatchObject({ code: "aborted" }); }); diff --git a/packages/chord/test/next/link/channel.test.ts b/packages/chord/test/next/link/channel.test.ts index f5abbe8eb..4179830af 100644 --- a/packages/chord/test/next/link/channel.test.ts +++ b/packages/chord/test/next/link/channel.test.ts @@ -20,11 +20,18 @@ import { RawPeer, rawLinked, rawProvider, + SERVER_ADDRESS, + Server, testCodec, transported, } from "../support.ts"; -const Thing = remote("app.thing", { version: 1, params: {}, variants: { only: { state: { n: int() } } } }); +const Thing = remote("app.thing", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { only: { state: { n: int() } } }, +}); const hold = { kind: "hold", @@ -33,6 +40,7 @@ const hold = { version: 1, fingerprint: "0000000000000000", params: "{}", + address: SERVER_ADDRESS, } as const; const problems = (errors: readonly HostError[]) => diff --git a/packages/chord/test/next/link/message.test.ts b/packages/chord/test/next/link/message.test.ts index 6adf76b4d..35be68873 100644 --- a/packages/chord/test/next/link/message.test.ts +++ b/packages/chord/test/next/link/message.test.ts @@ -21,12 +21,23 @@ function partText(part: Part): Part { } const incarnation = "0190f5a2-7c3e-7d4a-9b1c-2d3e4f5a6b7c"; + +/** The address a hold names: `Server()` on the wire. */ +const server = { kind: "server", params: {} }; const error = { kind: "chord", error: { code: "released", message: "gone" } } as const; /** One message of every kind, with the optional fields both present and absent; parts are JSON text. */ const EVERY_KIND: Message[] = [ { kind: "hello", protocol: 1, incarnation, calls: 256 }, - { kind: "hold", hold: 300, type: "app.notes", version: 2, fingerprint: "0123456789abcdef", params: '{"id":1}' }, + { + kind: "hold", + hold: 300, + type: "app.notes", + version: 2, + fingerprint: "0123456789abcdef", + params: '{"id":1}', + address: server, + }, { kind: "hold", hold: 1, @@ -45,6 +56,7 @@ const EVERY_KIND: Message[] = [ version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, incarnation, }, { kind: "release", hold: 0 }, @@ -88,6 +100,12 @@ const EVERY_KIND: Message[] = [ cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, retryIn: 0, }, + // Not the object's fault (dependency model §2): no backoff window, an event wakes it. + { + kind: "failed", + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "contract_mismatch", message: "m" } } }, + }, // The wire error format in `failed` and `gone`: a plugin's error travels too. { kind: "failed", hold: 0, cause: { kind: "broken", error: { kind: "plugin", code: "config", message: "m" } } }, { kind: "gone", hold: 1, error: { kind: "plugin", code: "config", message: "m", details: { at: 1 } } }, @@ -97,7 +115,9 @@ const EVERY_KIND: Message[] = [ cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "m" } } }, retryIn: 60_000, }, - { kind: "retry", hold: 300 }, + // With its hop budget (audit A5): the first host's, and the last one passed on. + { kind: "retry", hold: 300, hops: 16 }, + { kind: "retry", hold: 300, hops: 1 }, { kind: "loading", hold: 300 }, { kind: "result", call: 3, value: '{"n":1}' }, { kind: "result", call: 3 }, @@ -106,11 +126,20 @@ const EVERY_KIND: Message[] = [ { kind: "ack" }, { kind: "ack", origin: 1 }, { kind: "closed", origin: 300 }, - { kind: "hold", origin: 2, hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}" }, + { + kind: "hold", + origin: 2, + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + }, { kind: "release", origin: 2, hold: 3 }, { kind: "call", origin: 2, call: 1, hold: 3, method: 0, args: "[1]" }, { kind: "abort", origin: 2, call: 1 }, - { kind: "retry", origin: 2, hold: 3 }, + { kind: "retry", origin: 2, hold: 3, hops: 16 }, { kind: "bound", origin: 2, holds: [{ hold: 3, variant: 0, members: [], failed: [] }] }, { kind: "state", origin: 2, holds: [] }, { @@ -130,10 +159,20 @@ const EVERY_KIND: Message[] = [ version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, chain: [incarnation, incarnation], limit: 65_536, }, - { kind: "hold", hold: 4, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}", limit: 0 }, + { + kind: "hold", + hold: 4, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: "{}", + address: server, + limit: 0, + }, { kind: "revoke", revocation: { grant: "g1" } }, { kind: "ping" }, { kind: "pong" }, @@ -181,7 +220,15 @@ forEachCodec(() => { const codec = testCodec(); const deflated = () => new DeflatedPart(new Uint8Array([0, 1, 2, 255])); const messages: Message[] = [ - { kind: "hold", hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: deflated() }, + { + kind: "hold", + hold: 0, + type: "t", + version: 1, + fingerprint: "0123456789abcdef", + params: deflated(), + address: server, + }, { kind: "call", call: 0, hold: 0, method: 0, args: deflated() }, { kind: "result", call: 0, value: deflated(), segments: [new Uint8Array([7])] }, { @@ -253,6 +300,7 @@ forEachCodec(() => { version: 1, fingerprint: "0123456789abcdef", params: jsonPart({ "\uFEFFkey": "\uFEFFvalue" }), + address: server, }; const decoded = codec.decode(codec.encode(message)) as Extract; expect(plain(decoded)).toEqual(message); @@ -283,15 +331,31 @@ describe("the binary codec", () => { version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, }); - // kind, id, type, version, fingerprint, params, the optional fields' flags - expect(hold).toHaveLength(1 + 1 + 2 + 1 + 8 + 3 + 1); + // kind, id, type, version, fingerprint, params, the address's kind and params, the optional fields' flags + expect(hold).toHaveLength(1 + 1 + 2 + 1 + 8 + 3 + 7 + 3 + 1); expect([...binaryCodec.encode({ kind: "ack" })]).toEqual([13]); // An origin: the kind byte's high bit, then the origin as a varint. expect([...binaryCodec.encode({ kind: "ack", origin: 300 })]).toEqual([13 | 0x80, 0xac, 0x02]); expect([...binaryCodec.encode({ kind: "closed", origin: 1 })]).toEqual([14 | 0x80, 1]); }); + it("writes a failure's cause as one byte, 0 transient, 1 broken, 2 waiting (dependency model §2); refuses another", () => { + const error = { kind: "chord", error: { code: "unavailable", message: "m" } } as const; + const failed = (kind: "transient" | "broken" | "waiting"): Message => ({ + kind: "failed", + hold: 0, + cause: { kind, error }, + }); + expect( + ["transient", "broken", "waiting"].map((kind) => binaryCodec.encode(failed(kind as "waiting"))[2]), + ).toEqual([0, 1, 2]); + const unknown = binaryCodec.encode(failed("waiting")); + unknown[2] = 3; + expect(() => binaryCodec.decode(unknown)).toThrow("an unknown failure kind"); + }); + it("costs a state entry without retirements no byte: the failure count is doubled, plus one when they follow", () => { const state = (retired: number[]): Message => ({ kind: "state", @@ -345,6 +409,7 @@ describe("the binary codec", () => { version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, }); const forged = hold.slice(); forged[forged.length - 1] = 32; @@ -371,8 +436,8 @@ describe("the binary codec", () => { expect(() => binaryCodec.decode(new Uint8Array([1, 0, 1, 0xff]))).toThrow(); }); - it("writes retry and loading as their kind and hold, and failed's retryIn as one more than it (0: absent)", () => { - expect([...binaryCodec.encode({ kind: "retry", hold: 300 })]).toEqual([11, 0xac, 0x02]); + it("writes retry as its kind, hold and hops, loading as its kind and hold, and failed's retryIn as one more than it (0: absent)", () => { + expect([...binaryCodec.encode({ kind: "retry", hold: 300, hops: 16 })]).toEqual([11, 0xac, 0x02, 16]); expect([...binaryCodec.encode({ kind: "loading", hold: 1 })]).toEqual([12, 1]); const cause = { kind: "broken", error: { kind: "chord", error: { code: "released", message: "" } } } as const; const absent = binaryCodec.encode({ kind: "failed", hold: 0, cause }); @@ -392,7 +457,7 @@ describe("the text codec", () => { expect(text({ kind: "bound", holds: [{ hold: 0, variant: 1, members, failed: [] }] })).toBe( '["b",[[0,1,[[0,[0,5]]]]]]', ); - expect(text({ kind: "retry", hold: 2 })).toBe('["T",2]'); + expect(text({ kind: "retry", hold: 2, hops: 16 })).toBe('["T",2,16]'); expect(text({ kind: "loading", hold: 2 })).toBe('["l",2]'); const cause = { kind: "transient", @@ -404,6 +469,10 @@ describe("the text codec", () => { expect(text({ kind: "failed", hold: 2, cause, retryIn: 750 })).toBe( '["f",2,"t",{"kind":"chord","error":{"code":"unavailable","message":"m"}},750]', ); + expect(text({ kind: "failed", hold: 2, cause: { ...cause, kind: "waiting" } })).toBe( + '["f",2,"w",{"kind":"chord","error":{"code":"unavailable","message":"m"}}]', + ); + expect(() => textCodec.decode(new TextEncoder().encode('["f",2,"x",{}]'))).toThrow("an unknown failure kind"); expect(text({ kind: "ack" })).toBe('["a"]'); // An origin wraps the message's own tuple. expect(text({ kind: "release", origin: 4, hold: 3 })).toBe('["@",4,["R",3]]'); @@ -446,10 +515,12 @@ describe("the text codec", () => { version: 1, fingerprint: "0123456789abcdef", params: "{}", + address: server, } as const; - expect(text(hold)).toBe('["H",0,"t",1,"0123456789abcdef",[0,{}]]'); - expect(text({ ...hold, incarnation })).toBe(`["H",0,"t",1,"0123456789abcdef",[0,{}],null,null,"${incarnation}"]`); - expect(text({ ...hold, limit: 5 })).toBe('["H",0,"t",1,"0123456789abcdef",[0,{}],null,null,null,null,5]'); + const head = '["H",0,"t",1,"0123456789abcdef",[0,{}],["server",{}]'; + expect(text(hold)).toBe(`${head}]`); + expect(text({ ...hold, incarnation })).toBe(`${head},null,"${incarnation}"]`); + expect(text({ ...hold, limit: 5 })).toBe(`${head},null,null,null,5]`); }); it("puts byte segments after the tuple and a newline, their lengths the tuple's last field", () => { @@ -474,6 +545,23 @@ describe("the text codec", () => { expect(() => textCodec.decode(new TextEncoder().encode('["@",1,["@",2,["a"]]]'))).toThrow("wraps one message"); expect(() => textCodec.decode(new TextEncoder().encode('["@",1]'))).toThrow("wraps one message"); }); + + it("refuses a hold without its address: every hold names one", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}]]')).toThrow('a "H" message with 5 fields'); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}],null]')).toThrow("expected a list"); + expect(() => decode('["H",0,"t",1,"0123456789abcdef",[0,{}],["server"]]')).toThrow("an address is a pair"); + const hold = { kind: "hold", hold: 0, type: "t", version: 1, fingerprint: "0123456789abcdef", params: "{}" }; + expect(() => checkMessage(hold, context)).toThrow("message schema"); + }); + + it("refuses a retry without its hop budget, or with one outside 1..16 (audit A5)", () => { + const decode = (tuple: string) => textCodec.decode(new TextEncoder().encode(tuple)); + expect(() => decode('["T",2]')).toThrow('a "T" message with 1 fields'); + for (const hops of [0, 17]) { + expect(() => checkMessage({ kind: "retry", hold: 2, hops }, context)).toThrow("message schema"); + } + }); }); describe("describeMessage", () => { @@ -534,6 +622,7 @@ describe("the binary codec's short strings", () => { version: 1, fingerprint: "0123456789abcdef", params: jsonPart(type), + address: server, }; expect(plain(binaryCodec.decode(binaryCodec.encode(message)))).toEqual(message); }); diff --git a/packages/chord/test/next/link/values.test.ts b/packages/chord/test/next/link/values.test.ts index 84656b516..a92b5562d 100644 --- a/packages/chord/test/next/link/values.test.ts +++ b/packages/chord/test/next/link/values.test.ts @@ -29,6 +29,14 @@ describe("payloads", () => { expect(read.thumbs).toEqual(thumbs); }); + it("carry any Uint8Array subclass as bytes: a Node Buffer, whose toJSON the replacer never sees (slice 11)", () => { + const buffer = Buffer.from([1, 2, 3]); + const { json, segments } = payloadPart({ bytes: buffer, at: [buffer] }, true); + expect(json).toBe('{"bytes":{"$bytes":0},"at":[{"$bytes":1}]}'); + expect(segments![0]).toBe(buffer); + expect(payloadPart(buffer, true)).toEqual({ json: '{"$bytes":0}', segments: [buffer] }); + }); + it("marked as carrying bytes refuse the reserved key $bytes with invalid/args", () => { let thrown: unknown; try { diff --git a/packages/chord/test/next/link/websocket.test.ts b/packages/chord/test/next/link/websocket.test.ts index 821c1468e..c67212b48 100644 --- a/packages/chord/test/next/link/websocket.test.ts +++ b/packages/chord/test/next/link/websocket.test.ts @@ -4,7 +4,7 @@ import { WebSocket, WebSocketServer } from "ws"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { createHost, defineFacet } from "../../../src/next/host/index.ts"; import { type WebSocketLike, webSocketLink } from "../../../src/next/link/index.ts"; -import { Notes, serveNotes, type TestPrincipal, transported } from "../support.ts"; +import { Notes, Server, serveNotes, type TestPrincipal, transported } from "../support.ts"; /** A socket playing a WebSocket by hand. */ class FakeSocket implements WebSocketLike { @@ -93,7 +93,7 @@ describe("WebSocket links over a fake socket", () => { /** A `ws` server on a free loopback port whose connections a server host accepts, and a URL to it. */ async function loopback(options: { readonly maxPayload?: number } = {}) { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load([{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context); const wss = new WebSocketServer({ port: 0, host: "127.0.0.1", maxPayload: options.maxPayload }); await new Promise((resolve) => wss.once("listening", resolve)); diff --git a/packages/chord/test/next/registries/entries.test.ts b/packages/chord/test/next/registries/entries.test.ts new file mode 100644 index 000000000..88bd61003 --- /dev/null +++ b/packages/chord/test/next/registries/entries.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { + keyedEntries, + orderedEntries, + type RegistryConflict, + type RegistryTable, +} from "../../../src/next/registries/index.ts"; + +const table: RegistryTable<{ readonly routes?: Readonly>; readonly items?: readonly string[] }> = + [ + { plugin: "@a/one", version: 1, value: { routes: { "/x": "one", "/y": "one" }, items: ["late", "first"] } }, + { plugin: "@b/two", type: "b.store", version: 1, value: { routes: { "/x": "two" }, items: ["second"] } }, + { plugin: "@c/three", version: 1, value: {} }, + ]; + +describe("keyedEntries", () => { + it("maps every entry's keys with their plugin; a duplicate is reported and the first by plugin name keeps it", () => { + const conflicts: RegistryConflict[] = []; + const map = keyedEntries( + table, + (value) => value.routes, + (conflict) => conflicts.push(conflict), + ); + expect([...map]).toEqual([ + ["/x", { value: "one", plugin: "@a/one" }], + ["/y", { value: "one", plugin: "@a/one" }], + ]); + expect(conflicts).toEqual([{ message: '"/x" is contributed twice', plugins: ["@a/one", "@b/two"] }]); + }); +}); + +describe("orderedEntries", () => { + it("lists every entry's items in table order, with their plugin and type", () => { + expect(orderedEntries(table, (value) => value.items)).toEqual([ + { value: "late", plugin: "@a/one" }, + { value: "first", plugin: "@a/one" }, + { value: "second", plugin: "@b/two", type: "b.store" }, + ]); + }); + + it("sorts by `orderOf`, table order kept among equals", () => { + const order = (item: string) => (item === "first" ? 0 : 1); + expect(orderedEntries(table, (value) => value.items, order).map(({ value }) => value)).toEqual([ + "first", + "late", + "second", + ]); + }); +}); diff --git a/packages/chord/test/next/registries/record.test.ts b/packages/chord/test/next/registries/record.test.ts new file mode 100644 index 000000000..f9678df4b --- /dev/null +++ b/packages/chord/test/next/registries/record.test.ts @@ -0,0 +1,752 @@ +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { bool, int, local, method, obj } from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { createHost, defineFacet, type HostError } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { + defineRegistry, + keyedEntries, + type Registry, + type RegistryTable, + type TableItem, +} from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; +import type { NativeWatchCall } from "../../../src/watch.ts"; +import { + delivered, + expectCollected, + failures, + flushed, + instancesOf, + loadedPlugins, + metered, + storeRestartable, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** A registry whose composition is its table as `plugin:type=value` strings. */ +const Names = defineRegistry("app.names", { version: 1 }); + +const listed = (table: RegistryTable): readonly string[] => + table.map(({ plugin, type, value }) => `${plugin}:${type ?? ""}=${value}`); + +/** A host-lifetime object contributing a writable it sets with `set`. */ +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Setter` contributing a writable of `R`'s contribution type, `initial` first, `set(n)` makes `of(n)`. */ +function setter(R: Registry, initial: C, of: (n: number) => C) { + return (host: Parameters[0][string]>[0]) => + host.serve(Setter, (_, self) => { + const value = self.writable(initial); + self.contribute(R, value); + return { + set: async (n: number) => { + value.set(of(n)); + return null; + }, + }; + }); +} + +function kinds(errors: readonly HostError[]): string[] { + return errors.map((error) => error.kind); +} + +describe("the table", () => { + it("is sorted by plugin name, then type, a facet first: never load order", async () => { + const { host } = await loadedPlugins({ + "@z/late": (_, self) => self.contribute(Names, "z"), + "@a/early": (host, self) => { + self.contribute(Names, "a"); + setter(Names, "store", String)(host); + }, + "@m/provider": (host) => host.serve(Names, { compose: listed }), + }); + const value = await host.registry(Names, context); + expect(value.value).toEqual(["@a/early:=a", "@a/early:app.setter=store", "@z/late:=z"]); + await host.close(context); + }); + + it("adapts an older contribution once per value; a newer one, or an older one without an adapter, is left out and reported once", async () => { + const V3 = defineRegistry("app.versions", { version: 3 }); + const V2 = defineRegistry<{ old: string }, unknown>("app.versions", { version: 2 }); + const V1 = defineRegistry("app.versions", { version: 1 }); + const V4 = defineRegistry("app.versions", { version: 4 }); + const adapted: string[] = []; + const { host, errors } = await loadedPlugins({ + "@app/provider": (host) => + host.serve(V3, { + compose: listed, + accepts: { + 2: { + contribution: (old: { old: string }) => { + adapted.push(old.old); + return `adapted ${old.old}`; + }, + }, + }, + }), + "@app/one": (_, self) => self.contribute(V1, "one"), + "@app/two": setter(V2, { old: "first" }, (n) => ({ old: `${n}` })), + "@app/three": (_, self) => self.contribute(V3, "three"), + "@app/four": (_, self) => self.contribute(V4, "four"), + }); + const value = await host.registry(V3, context); + expect(value.value).toEqual(["@app/three:=three", "@app/two:app.setter=adapted first"]); + expect(errors).toEqual([ + { kind: "contribution_refused", registry: "app.versions", plugin: "@app/four", version: 4 }, + { kind: "contribution_refused", registry: "app.versions", plugin: "@app/one", version: 1 }, + ]); + const setters = await host.get(Setter, {}, context); + await setters.set(5, context); + expect(value.value).toEqual(["@app/three:=three", "@app/two:app.setter=adapted 5"]); + // Each value adapted once, though the table was built again; no refusal reported twice. + expect(adapted).toEqual(["first", "5"]); + expect(kinds(errors)).toEqual(["contribution_refused", "contribution_refused"]); + await host.close(context); + }); + + it("takes a contribution declared before its provider's setup into the first composition", async () => { + let composed: readonly string[] | undefined; + const { host } = await loadedPlugins({ + "@app/a": (_, self) => self.contribute(Names, "first"), + "@app/provider": (host) => + host.serve(Names, { + compose: (table) => { + composed = listed(table); + return composed; + }, + }), + }); + expect(composed).toEqual(["@app/a:=first"]); + await host.close(context); + }); +}); + +describe("compose", () => { + it("composes every served registry once at the end of load's setups, an empty table too, before any object activates", async () => { + const { telemetry, counts } = await metered(); + const Other = defineRegistry("app.other", { version: 1 }); + const Reader = local("app.reader", { version: 1, params: {}, members: { seen: int() } }); + let seen: readonly string[] | undefined; + const { host } = await loadedPlugins( + { + "@app/provider": (host, self) => { + host.serve(Names, { compose: listed }); + host.serve(Other, { compose: (table) => table.length }); + self.contribute(Names, "a"); + host.serve(Reader, (_, self) => { + const names = self.registry(Names); + self.open( + async () => { + seen = names.value; + }, + () => {}, + ); + return { seen: self.derive([self.registry(Other)], ([n]) => n) }; + }); + }, + }, + { telemetry }, + ); + expect(seen).toEqual(["@app/provider:=a"]); + expect((await host.get(Reader, {}, context)).seen.value).toBe(0); + expect(counts()["chord.registry.composes"]).toBe(2); + await host.close(context); + }); + + it("is lazy: a change marks the table, a read composes it once; no compose for an unchanged table", async () => { + const { telemetry, counts } = await metered(); + const Flags = defineRegistry("app.flags", { version: 1 }); + const Doc = defineState("app.flag", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 1 }) }); + const Flagger = local("app.flagger", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins( + { + "@app/flags": (host) => { + host.serve(Flags, { compose: (table) => table.filter(({ value }) => value).length }); + host.serve(Flagger, (_, self) => { + const doc = self.state(Doc); + self.contribute( + Flags, + self.derive([doc], ([d]) => d.n > 0), + ); + return { + set: async (n: number, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + }, + }, + { telemetry }, + ); + const flags = await host.registry(Flags, context); + const flagger = await host.get(Flagger, {}, context); + expect(flags.value).toBe(1); + const base = counts()["chord.registry.composes"]!; + expect(flags.value).toBe(1); + expect(counts()["chord.registry.composes"]).toBe(base); + // The derive changes, its value does not: the table stays, nothing composes. + await flagger.set(2, context); + await flagger.set(3, context); + expect(flags.value).toBe(1); + expect(counts()["chord.registry.composes"]).toBe(base); + // Nobody reads it: nothing composes until a read. + await flagger.set(-1, context); + await flagger.set(-2, context); + expect(counts()["chord.registry.composes"]).toBe(base); + expect(flags.value).toBe(0); + expect(counts()["chord.registry.composes"]).toBe(base + 1); + await host.close(context); + }); + + it("composes a watched registry in the batch of the commit that changed a contribution, once per batch", async () => { + const { telemetry, counts } = await metered(); + const Doc = defineState("app.names", { + schema: obj({ a: int(), b: int() }), + version: 1, + initial: () => ({ a: 0, b: 0 }), + }); + const Two = local("app.two", { version: 1, params: {}, members: { both: method(int()), sum: int() } }); + const Sum = defineRegistry<{ readonly a: number; readonly b: number }, number>("app.sum", { version: 1 }); + const { host } = await loadedPlugins( + { + "@app/sum": (host) => { + host.serve(Sum, { compose: (table) => table.reduce((n, { value }) => n + value.a + value.b, 0) }); + host.serve(Two, (_, self) => { + const doc = self.state(Doc); + self.contribute(Sum, doc); + return { + sum: self.derive([self.registry(Sum)], ([n]) => n), + both: async (n: number, ctx) => { + await self.tx(async (tx) => { + const d = await tx.draft(doc); + d.a = n; + d.b = n; + }, ctx); + return null; + }, + }; + }); + }, + }, + { telemetry }, + ); + const two = await host.get(Two, {}, context); + const sums: number[] = []; + const watch = await two.sum.watch(context); + watch.start(async (call) => { + if (call.kind === "value") sums.push(call.value); + }); + const base = counts()["chord.registry.composes"]!; + await two.both(2, context); + await delivered(); + // One commit, both fields: one compose in its batch, one frame of the derive over it. + expect(sums).toEqual([4]); + expect(counts()["chord.registry.composes"]).toBe(base + 1); + await host.close(context); + }); + + it("reports a conflict once per episode", async () => { + const Routes = defineRegistry>, ReadonlyMap>>( + "app.routes", + { version: 1 }, + ); + const { host, errors } = await loadedPlugins({ + "@app/a": (_, self) => self.contribute(Routes, { "/x": "a" }), + "@app/b": setter(Routes, { "/x": "b" } as Readonly>, (n) => ({ + "/x": "b", + [`/${n}`]: "b", + })), + "@app/http": (host) => + host.serve(Routes, { compose: (table, { report }) => keyedEntries(table, (c) => c, report) }), + }); + const routes = await host.registry(Routes, context); + expect(routes.value.get("/x")).toEqual({ value: "a", plugin: "@app/a" }); + const setters = await host.get(Setter, {}, context); + await setters.set(1, context); + void routes.value; + await setters.set(2, context); + void routes.value; + expect(errors).toEqual([ + { + kind: "registry_conflict", + registry: "app.routes", + message: '"/x" is contributed twice', + plugins: ["@app/a", "@app/b"], + }, + ]); + await host.close(context); + }); + + it("a throwing compose: reported once, a failed call, `.value` keeps the last good composition; the next good one heals", async () => { + const Count = defineRegistry("app.count", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/count": (host) => { + host.serve(Count, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error(`negative: ${sum}`); + return sum; + }, + }); + setter(Count, 1, (n) => n)(host); + }, + }); + const count = await host.registry(Count, context); + const calls: NativeWatchCall[] = []; + const watch = await count.watch(context); + watch.start(async (call) => void calls.push(call)); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + await setters.set(-2, context); + await delivered(); + expect(count.value).toBe(1); + expect(errors).toEqual([{ kind: "registry_failed", registry: "app.count", error: new Error("negative: -1") }]); + await setters.set(1, context); + await delivered(); + // The heal is a value call, also when its value is the last good one. + expect(calls).toEqual([ + { kind: "failed", error: new Error("negative: -1") }, + { kind: "value", value: 1 }, + ]); + await setters.set(3, context); + await delivered(); + expect(calls.at(-1)).toEqual({ kind: "value", value: 3 }); + await host.close(context); + }); + + it("before any good composition, `.value` throws the error and `host.registry` rejects with it at its readyTimeout", async () => { + const Broken = defineRegistry("app.broken", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/broken": (host) => + host.serve(Broken, { + compose: () => { + throw new Error("bug"); + }, + }), + }); + await expect(host.registry(Broken, context, { readyTimeout: 5 })).rejects.toThrow("bug"); + expect(kinds(errors)).toEqual(["registry_failed"]); + await host.close(context); + }); +}); + +describe("contributions", () => { + it("a watchable contribution that fails is left out and reported once, and is back once it heals", async () => { + const Doc = defineState("app.n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 1 }) }); + const Failing = local("app.failing", { version: 1, params: {}, members: { set: method(int()) } }); + const { host, errors } = await loadedPlugins({ + "@app/names": (host, self) => { + host.serve(Names, { compose: listed }); + self.contribute(Names, "facet"); + host.serve(Failing, (_, self) => { + const doc = self.state(Doc); + self.contribute( + Names, + self.derive([doc], ([d]) => { + if (d.n < 0) throw new Error("negative"); + return `n${d.n}`; + }), + ); + return { + set: async (n: number, ctx) => { + await doc.change((d) => { + d.n = n; + }, ctx); + return null; + }, + }; + }); + }, + }); + const names = await host.registry(Names, context); + const failing = await host.get(Failing, {}, context); + expect(names.value).toEqual(["@app/names:=facet", "@app/names:app.failing=n1"]); + await failing.set(-1, context); + expect(names.value).toEqual(["@app/names:=facet"]); + await failing.set(-2, context); + expect(names.value).toEqual(["@app/names:=facet"]); + await failing.set(2, context); + expect(names.value).toEqual(["@app/names:=facet", "@app/names:app.failing=n2"]); + expect(errors.filter(({ kind }) => kind === "contribution_failed")).toEqual([ + { + kind: "contribution_failed", + registry: "app.names", + plugin: "@app/names", + type: "app.failing", + error: new Error("negative"), + }, + ]); + await host.close(context); + }); + + it("a contribution adapter that throws leaves that contribution out (`contribution_failed`)", async () => { + const V2 = defineRegistry("app.adapted", { version: 2 }); + const V1 = defineRegistry("app.adapted", { version: 1 }); + const { host, errors } = await loadedPlugins({ + "@app/provider": (host, self) => { + host.serve(V2, { + compose: listed, + accepts: { + 1: { + contribution: () => { + throw new Error("cannot adapt"); + }, + }, + }, + }); + self.contribute(V2, "current"); + }, + "@app/old": (_, self) => self.contribute(V1, "old"), + }); + expect((await host.registry(V2, context)).value).toEqual(["@app/provider:=current"]); + expect(errors).toEqual([ + { kind: "contribution_failed", registry: "app.adapted", plugin: "@app/old", error: new Error("cannot adapt") }, + ]); + await host.close(context); + }); + + it("catches a ring of registries at its first composition: the contributors go live, the rest works (audit A2)", async () => { + const A = defineRegistry("app.ring-a", { version: 1 }); + const B = defineRegistry("app.ring-b", { version: 1 }); + const ToA = local("app.to-a", { version: 1, params: {}, members: {} }); + const ToB = local("app.to-b", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/ring": (h, self) => { + h.serve(A, { compose: (table) => table.length }); + h.serve(B, { compose: (table) => table.length }); + h.serve(ToA, (_, self) => { + self.contribute(A, self.registry(B)); + return {}; + }); + h.serve(ToB, (_, self) => { + self.contribute(B, self.registry(A)); + return {}; + }); + self.contribute(A, "plain"); + self.contribute(B, "plain"); + }, + }); + const live = [await host.get(ToA, {}, context), await host.get(ToB, {}, context)]; + expect(live.map((reference) => reference.state.value)).toEqual([{ kind: "live" }, { kind: "live" }]); + // A composes first: B composes for it and its contribution of A, read while A composes, is left out. A has its + // plain entry and B's value, B its plain entry. + expect([(await host.registry(A, context)).value, (await host.registry(B, context)).value]).toEqual([2, 1]); + expect(errors).toEqual([ + { + kind: "contribution_failed", + registry: "app.ring-b", + plugin: "@app/ring", + type: "app.to-b", + error: expect.objectContaining({ code: "invalid", reason: "cycle" }), + }, + ]); + await host.close(context); + }); + + it("keeps a ring's later recompositions bounded: a change composes each registry once, then nothing (audit A2)", async () => { + const A = defineRegistry("app.later-a", { version: 1 }); + const B = defineRegistry("app.later-b", { version: 1 }); + const Ring = local("app.later-ring", { version: 1, params: {}, members: {} }); + const Back = local("app.later-back", { version: 1, params: {}, members: {} }); + const Extra = local("app.later-extra", { version: 1, params: {}, members: {} }); + let composes = 0; + let extra: Writable | undefined; + const sum = (table: RegistryTable): number => { + composes++; + return table.reduce((total, { value }) => total + (typeof value === "number" ? value : 0), 0); + }; + const { host, errors } = await loadedPlugins({ + "@app/later": (h) => { + h.serve(A, { compose: sum }); + h.serve(B, { compose: sum }); + h.serve(Ring, (_, self) => { + self.contribute(A, self.registry(B)); + return {}; + }); + h.serve(Back, (_, self) => { + self.contribute(B, self.registry(A)); + return {}; + }); + h.serve(Extra, (_, self) => { + extra = self.writable(1); + self.contribute(B, extra); + return {}; + }); + }, + }); + await host.get(Ring, {}, context); + await host.get(Back, {}, context); + await host.get(Extra, {}, context); + const a = await host.registry(A, context); + const b = await host.registry(B, context); + const watches = [await a.watch(context), await b.watch(context)]; + await flushed(); + const before = composes; + extra!.set(5); + await flushed(); + // B changed and composes first now: it reads A, whose contribution of B, read while B composes, is cut instead. + expect(composes - before).toBe(2); + expect([a.value, b.value]).toEqual([0, 5]); + expect(errors.map((error) => (error.kind === "contribution_failed" ? error.registry : error.kind))).toEqual([ + "app.later-b", + "app.later-a", + ]); + // Nothing composes on by itself. + await flushed(); + expect(composes - before).toBe(2); + for (const watch of watches) await watch.stop(); + await host.close(context); + }); + + it("catches a ring through a derive, and one declared within one factory, the same way (audit A2)", async () => { + const A = defineRegistry("app.both-a", { version: 1 }); + const B = defineRegistry("app.both-b", { version: 1 }); + const Both = local("app.both", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/both": (h) => { + h.serve(A, { compose: (table) => table.length }); + h.serve(B, { compose: (table) => table.length }); + h.serve(Both, (_, self) => { + self.contribute(A, self.registry(B)); + self.contribute( + B, + self.derive([self.registry(A)], ([n]) => n as number), + ); + return {}; + }); + }, + }); + expect((await host.get(Both, {}, context)).state.value).toEqual({ kind: "live" }); + await host.registry(A, context); + await host.registry(B, context); + const cut = errors.filter((error) => error.kind === "contribution_failed"); + expect(cut.length).toBeGreaterThan(0); + for (const error of cut) expect(error).toMatchObject({ error: { code: "invalid", reason: "cycle" } }); + expect(failures(errors)).toEqual([]); + await host.close(context); + }); + + it("follows only Chord watchables: anything else is a plain value, whatever methods it has (Sol)", async () => { + const Natives = defineRegistry<{ watch(): string; readonly value: string }, readonly string[]>("app.natives", { + version: 1, + }); + const native = { value: "a handle", watch: () => "a native operation" }; + const { host } = await loadedPlugins({ + "@app/x": (h, self) => { + h.serve(Natives, { compose: (table) => table.map(({ value }) => value.watch()) }); + self.contribute(Natives, native); + }, + }); + expect((await host.registry(Natives, context)).value).toEqual(["a native operation"]); + await host.close(context); + }); + + it("keeps an `undefined` contribution in the table (Sol, Astra)", async () => { + const Nothing = defineRegistry("app.undefined", { version: 1 }); + const { host } = await loadedPlugins({ + "@app/a": (h, self) => { + h.serve(Nothing, { compose: (table) => table.length }); + self.contribute(Nothing, undefined); + }, + }); + expect((await host.registry(Nothing, context)).value).toBe(1); + await host.close(context); + }); + + it("leaves out a registry's value contributed to another while that registry's compose throws (Sol, Astra)", async () => { + const Source = defineRegistry("app.source", { version: 1 }); + const Target = defineRegistry("app.target", { version: 1 }); + const Bridge = local("app.bridge", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Source, { + compose: (table) => { + const n = table.reduce((sum, { value }) => sum + value, 0); + if (n < 0) throw new Error("negative"); + return n; + }, + }); + h.serve(Target, { compose: (table) => table.map(({ value }) => value) }); + setter(Source, 1, (n) => n)(h); + h.serve(Bridge, (_, self) => { + self.contribute(Target, self.registry(Source)); + return {}; + }); + }, + }); + const target = await host.registry(Target, context); + expect(target.value).toEqual([1]); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + // The source's readers keep its last good value; as a contribution it is failed: left out, reported. + expect((await host.registry(Source, context)).value).toBe(1); + expect(target.value).toEqual([]); + expect(errors.find((error) => error.kind === "contribution_failed")).toMatchObject({ + registry: "app.target", + plugin: "@app/a", + type: "app.bridge", + error: new Error("negative"), + }); + await setters.set(3, context); + expect(target.value).toEqual([3]); + await host.close(context); + }); + + it("a followed contribution whose runtime ended its observer stays with its last value (R4: the source keeps it)", async () => { + let end: (() => void) | undefined; + const Follower = local("app.follower", { version: 1, params: {}, members: {} }); + const { host, errors } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Names, { compose: listed }); + h.serve(Follower, (_, self) => { + self.contribute( + Names, + self.follow(async (observer) => { + end = () => observer.end({ kind: "ended" }); + return { value: "followed", detach: () => {} }; + }), + ); + return {}; + }); + }, + }); + const names = await host.registry(Names, context); + expect(names.value).toEqual(["@app/a:app.follower=followed"]); + end!(); + await flushed(); + expect(names.value).toEqual(["@app/a:app.follower=followed"]); + expect(errors.filter(({ kind }) => kind === "contribution_failed")).toEqual([]); + await host.close(context); + }); + + it("an object's contribution is in the table from live until it fails, again once retried, until it ends", async () => { + const { host, fail, retry } = await storeRestartable( + objects, + (attempt, self) => self.contribute(Names, `attempt ${attempt}`), + { serve: (host) => host.serve(Names, { compose: listed }) }, + ); + const names = await host.registry(Names, context); + expect(names.value).toEqual(["@test/app:app.store=attempt 1"]); + fail(); + expect(names.value).toEqual([]); + await retry(); + expect(names.value).toEqual(["@test/app:app.store=attempt 2"]); + await host.close(context); + // Closed: nothing composes any more; a value a reader keeps reads the last composition. + expect(names.value).toEqual(["@test/app:app.store=attempt 2"]); + }); +}); + +describe("the last good composition (decision 5)", () => { + it("is dropped when a contribution it was built from leaves: `.value` throws until `compose` succeeds", async () => { + const Count = defineRegistry("app.count", { version: 1 }); + const compose = (table: RegistryTable): number => { + if (table.length === 0) throw new Error("no contributor"); + return table[0]!.value; + }; + const { host, fail, retry } = await storeRestartable( + objects, + (attempt, self) => self.contribute(Count, attempt), + { + serve: (host) => host.serve(Count, { compose }), + }, + ); + const count = await host.registry(Count, context); + expect(count.value).toBe(1); + // Its contributor failed: the last good value was built from its contribution (a native may have closed with it). + fail(); + expect(() => count.value).toThrow("no contributor"); + await retry(); + expect(count.value).toBe(2); + await host.close(context); + }); +}); + +describe("the table's memory", () => { + it("keeps nothing of a closed host: records, tables and followed contributions go with it", async () => { + const Doc = defineState("app.kept", { schema: obj({ on: bool() }), version: 1, initial: () => ({ on: true }) }); + const Kept = local("app.kept", { version: 1, params: {}, members: {} }); + const refs = await (async () => { + const { host } = await loadedPlugins({ + "@app/kept": (host) => { + host.serve(Names, { compose: listed }); + host.serve(Kept, (_, self) => { + self.contribute( + Names, + self.derive([self.state(Doc)], ([d]) => `${d.on}`), + ); + return {}; + }); + }, + }); + const names = new WeakRef(await host.registry(Names, context)); + await host.close(context); + return [names, new WeakRef(host)]; + })(); + await expectCollected(...refs); + }); +}); + +describe("a reload's new contribution adapter (fix round 1)", () => { + it("remaps an unchanged contributor with the new provider's adapter", async () => { + const R = defineRegistry("astra.contrib", { version: 2 }); + const Old = defineRegistry("astra.contrib", { version: 1 }); + const contribution = defineFacet((_, self) => self.contribute(Old, 7)); + const provider = (factor: number) => + defineFacet((h) => + h.serve(R, { + compose: (entries) => entries.reduce((n, e) => n + e.value, 0), + accepts: { 1: { contribution: (n) => n * factor, value: (n) => n } }, + }), + ); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@astra/provider", facet: provider(2) }, + { name: "@astra/contribution", facet: contribution }, + ], + context, + ); + const value = await host.registry(R, context); + expect(value.value).toBe(14); + await host.reload( + [ + { name: "@astra/provider", facet: provider(3) }, + { name: "@astra/contribution", facet: contribution }, + ], + context, + ); + const actual = value.value; + await host.close(context); + expect(actual).toBe(21); + }); +}); + +/** A registry its plugin serves: never listed as undeclared. */ +const Served = defineRegistry("app.served", { version: 1 }); + +describe("registries without a provider (slice 12 §6, testPlugin)", () => { + it("lists a registry with contributions and no provider, with the newest version contributed", async () => { + const Older = defineRegistry("app.names", { version: 1 }); + const Newer = defineRegistry("app.names", { version: 3 }); + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "a", facet: defineFacet((_host, self) => self.contribute(Older, "a")) }, + { name: "b", facet: defineFacet((_host, self) => self.contribute(Newer, "b")) }, + { name: "c", facet: defineFacet((h) => h.serve(Served, { compose: () => [] })) }, + ], + context, + ); + expect((host as unknown as ChordHost).undeclaredRegistries()).toEqual([{ name: "app.names", version: 3 }]); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/registries/registry.test.ts b/packages/chord/test/next/registries/registry.test.ts new file mode 100644 index 000000000..f1d31373e --- /dev/null +++ b/packages/chord/test/next/registries/registry.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet } from "../../../src/next/host/index.ts"; +import { defineRegistry, type Registry, type RegistryValue } from "../../../src/next/registries/index.ts"; +import { loadedPlugins } from "../support.ts"; + +describe("tokens", () => { + it("carry a name, a version and the types only; names and versions are checked at declaration", () => { + const Http = defineRegistry<{ readonly path: string }, readonly string[]>("chord.http", { version: 2 }); + expect(Http).toEqual({ kind: "registry", name: "chord.http", version: 2 }); + expectTypeOf(Http).toEqualTypeOf>(); + expect(() => defineRegistry("Chord.Http", { version: 1 })).toThrow('registry "Chord.Http": expected lowercase'); + expect(() => defineRegistry("chord.http", { version: 0 })).toThrow( + "chord.http.version: expected a positive integer", + ); + }); +}); + +describe("providers", () => { + const Names = defineRegistry("app.names", { version: 1 }); + + it("one per host: a second serve leaves its plugin out at load, reported", async () => { + const host = createHost({ onError: () => {} }); + const serve = defineFacet((h) => h.serve(Names, { compose: (table) => table.length })); + const { failed } = await host.load( + [ + { name: "@app/a", facet: serve }, + { name: "@app/b", facet: serve }, + ], + context, + ); + expect(failed).toEqual([ + { + plugin: "@app/b", + error: expect.objectContaining({ + message: 'plugin "@app/b": registry app.names is served twice (plugin "@app/a")', + }), + }, + ]); + await host.close(context); + }); + + it("a provider without compose leaves its plugin out at load (plugin code is not type-checked)", async () => { + const { host, failed } = await loadedPlugins({ "@app/a": (h) => h.serve(Names, {} as never) }); + expect(failed).toEqual([ + { + plugin: "@app/a", + error: expect.objectContaining({ message: 'plugin "@app/a": registry app.names: compose is a function' }), + }, + ]); + await host.close(context); + }); + + it("a registry whose provider comes with `load` heals a `host.registry` made before it", async () => { + const host = createHost({ onError: () => {} }); + const waiting = host.registry(Names, context); + await host.load( + [{ name: "@app/a", facet: defineFacet((h) => h.serve(Names, { compose: (t) => t.length })) }], + context, + ); + const value = await waiting; + expectTypeOf(value).toEqualTypeOf>(); + expect(value.value).toBe(0); + await host.close(context); + }); + + it("are served only while the setup runs", async () => { + let later: Parameters[0]>[0] | undefined; + const { host } = await loadedPlugins({ + "@app/a": (h) => { + later = h; + }, + }); + expect(() => later!.serve(Names, { compose: (t) => t.length })).toThrow( + "@app/a: host.serve is valid only in the facet's setup", + ); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/registries/value.test.ts b/packages/chord/test/next/registries/value.test.ts new file mode 100644 index 000000000..9b102ebe2 --- /dev/null +++ b/packages/chord/test/next/registries/value.test.ts @@ -0,0 +1,540 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal, withCancel } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import type { FollowObserver } from "../../../src/next/host/follow.ts"; +import { createHost, defineFacet, type Native } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry, type Registry } from "../../../src/next/registries/index.ts"; +import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; +import { + expectCollected, + failures, + flushed, + instancesOf, + loadedPlugins, + metered, + rejection, + settledHeap, +} from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +/** The provider's version 2: contributions are numbers, the value their sum. */ +const Sum = defineRegistry("app.sum", { version: 2 }); + +/** A reader built against version 1: its value was a plain number. */ +const SumV1 = defineRegistry("app.sum", { version: 1 }); + +/** A reader built against a version the provider does not know yet. */ +const SumV3 = defineRegistry("app.sum", { version: 3 }); + +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Setter`, contributing a writable number to `Sum`. */ +function setter(host: Parameters[0]>[0], initial = 1): void { + host.serve(Setter, (_, self) => { + const value = self.writable(initial); + self.contribute(Sum, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); +} + +const total = (table: readonly { readonly value: number }[]) => ({ + total: table.reduce((n, { value }) => n + value, 0), +}); + +describe("older readers (S2)", () => { + it("get the composition mapped back with accepts[old].value, once per composition", async () => { + const { telemetry, counts } = await metered(); + const mapped: number[] = []; + const { host } = await loadedPlugins( + { + "@app/sum": (host) => { + host.serve(Sum, { + compose: total, + accepts: { + 1: { + value: (current) => { + mapped.push(current.total); + return current.total; + }, + }, + }, + }); + setter(host); + }, + }, + { telemetry }, + ); + const old = await host.registry(SumV1, context); + const current = await host.registry(Sum, context); + expect(old.value).toBe(1); + expect(old.value).toBe(1); + expect(current.value).toEqual({ total: 1 }); + await (await host.get(Setter, {}, context)).set(4, context); + expect(old.value).toBe(4); + expect(mapped).toEqual([1, 4]); + expect(counts()["chord.registry.adapts"]).toBe(2); + await host.close(context); + }); + + it("without a value mapping, and for a reader newer than the provider, park readers failed { waiting } (contract_mismatch)", async () => { + const Newer = local("app.newer", { version: 1, params: {}, members: { total: int() } }); + const Older = local("app.older", { version: 1, params: {}, members: { total: int() } }); + const { host, errors } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { compose: total }); + host.serve(Newer, (_, self) => ({ total: self.derive([self.registry(SumV3)], ([v]) => v.total) })); + host.serve(Older, (_, self) => ({ total: self.derive([self.registry(SumV1)], ([v]) => v) })); + }, + }); + const quick = { readyTimeout: 5 }; + expect(await rejection(host.registry(SumV1, context, quick))).toMatchObject({ code: "contract_mismatch" }); + expect(await rejection(host.registry(SumV3, context, quick))).toMatchObject({ code: "contract_mismatch" }); + // Not their fault: waiting (a reload serving the version wakes them), no backoff, nothing failed. + expect(failures(errors)).toEqual([]); + expect(host.hold(Newer, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { + message: "waiting for registry app.sum: registry app.sum: version 3 is not served here (version 2 is)", + }, + }, + retryAt: undefined, + }); + await host.close(context); + }); + + it("drops an older reader's last good mapping after a contributor it was built from failed live (fix round 1, Astra 2)", async () => { + const R = defineRegistry("app.mapped", { version: 2 }); + const Old = defineRegistry("app.mapped", { version: 1 }); + const Contributor = local("app.contributor", { version: 1, params: {}, members: { n: int() } }); + let observer: FollowObserver | undefined; + const { host } = await loadedPlugins({ + "@app/mapping": (h) => { + h.serve(R, { + compose: (table) => table.reduce((n, e) => n + e.value, 0), + accepts: { + 1: { + value: (n) => { + if (n === 0) throw new Error("old version cannot map the empty table"); + return n; + }, + }, + }, + }); + h.serve(Contributor, (_, self) => { + self.contribute(R, 42); + return { + n: self.follow(async (followed) => { + observer = followed; + return { value: 42, detach() {} }; + }), + }; + }); + }, + }); + const old = await host.registry(Old, context); + expect(old.value).toBe(42); + observer!.end({ kind: "failed", error: new Error("runtime ended") }); + await flushed(); + expect(() => old.value).toThrow("old version cannot map the empty table"); + await host.close(context); + }); + + it("never hands out a mapped native whose contributor failed and closed it (fix round 1, Sol 2)", async () => { + type Resource = { closed: boolean }; + const Current = defineRegistry, readonly Native[]>("app.resources", { version: 2 }); + const Old = defineRegistry, Native>("app.resources", { version: 1 }); + const Owner = local("app.owner", { version: 1, params: {}, members: {} }); + const resource = { closed: false }; + let failRuntime: () => void = () => {}; + const { host } = await loadedPlugins({ + "@app/resources": (h) => { + h.serve(Current, { + compose: (table) => table.map(({ value }) => value), + accepts: { + 1: { + value: (current) => { + if (current.length === 0) throw new Error("no current resource"); + return current[0]!; + }, + }, + }, + }); + h.serve(Owner, (_, self) => { + const native = self.open( + async () => resource, + () => { + resource.closed = true; + }, + ); + self.contribute(Current, native); + self.follow(async (observer) => { + failRuntime = () => observer.end({ kind: "failed", error: new Error("runtime closed") }); + return { value: 1, detach: () => {} }; + }); + return {}; + }); + }, + }); + const value = await host.registry(Old, context); + expect(value.value.value).toBe(resource); + failRuntime(); + await flushed(); + expect(resource.closed).toBe(true); + expect(() => value.value).toThrow("no current resource"); + await host.close(context); + }); + + it("maps an `undefined` composition (Sol, Astra)", async () => { + const Nothing = defineRegistry("app.nothing", { version: 2 }); + const NothingV1 = defineRegistry("app.nothing", { version: 1 }); + const { host } = await loadedPlugins({ + "@app/a": (h) => h.serve(Nothing, { compose: () => undefined, accepts: { 1: { value: () => "mapped" } } }), + }); + expect((await host.registry(NothingV1, context)).value).toBe("mapped"); + await host.close(context); + }); + + it("a failed mapping heals when the composition is the one mapped before (Astra)", async () => { + const Count = defineRegistry("app.count", { version: 2 }); + const CountV1 = defineRegistry("app.count", { version: 1 }); + const Counter = local("app.counter", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins({ + "@app/count": (h) => { + h.serve(Count, { + compose: (table) => table.reduce((n, { value }) => n + value, 0), + accepts: { + 1: { + value: (n) => { + if (n < 0) throw new Error("negative"); + return n; + }, + }, + }, + }); + h.serve(Counter, (_, self) => { + const value = self.writable(1); + self.contribute(Count, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }, + }); + const old = await host.registry(CountV1, context); + const calls: NativeWatchCall[] = []; + old.observe({ advance: (call) => calls.push(call), end: () => {} }, context); + const counter = await host.get(Counter, {}, context); + await counter.set(-1, context); + // Back to the composition mapped before (the same number): the mapping heals. + await counter.set(1, context); + expect(calls).toEqual([ + { kind: "failed", error: new Error("negative") }, + { kind: "value", value: 1 }, + ]); + await host.close(context); + }); +}); + +describe("readers (S3)", () => { + const Reader = local("app.reader", { version: 1, params: {}, members: { total: int() } }); + const serveReader = ( + host: Parameters[0]>[0], + R: Registry = Sum, + ) => host.serve(Reader, (_, self) => ({ total: self.derive([self.registry(R)], ([v]) => v.total) })); + + it("activate once the registry has a value; a reader of a registry nobody serves parks failed { waiting }, `load` waits for nothing", async () => { + const started = Date.now(); + const { host, errors } = await loadedPlugins({ "@app/reader": (host) => serveReader(host) }); + expect(Date.now() - started).toBeLessThan(1_000); + expect(errors).toEqual([]); + expect(host.hold(Reader, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { + kind: "waiting", + error: { + code: "unavailable", + message: "waiting for registry app.sum: registry app.sum: no plugin serves it here", + cause: { code: "unavailable", reason: "undeclared" }, + }, + }, + }); + expect(await rejection(host.registry(Sum, context, { readyTimeout: 5 }))).toMatchObject({ + code: "unavailable", + reason: "undeclared", + }); + await host.close(context); + }); + + it("a reader parked on a first `compose` that threw goes on at the first good composition", async () => { + const { host, errors } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total === 0) throw new Error("empty"); + return sum; + }, + }); + serveReader(host); + }, + "@app/setter": (host) => setter(host, 0), + }); + expect(failures(errors)).toEqual([]); + const held = host.hold(Reader, {}, context); + expect(held.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + const setters = await host.get(Setter, {}, context); + await setters.set(3, context); + // Woken by the composition: no timer of its own. + const reader = await host.get(Reader, {}, context, { readyTimeout: 500 }); + held.release(); + expect(reader.total.value).toBe(3); + await host.close(context); + }); + + it("host.registry before the publication waits for it, then resolves with the provider it served", async () => { + const host = createHost({ onError: () => {} }); + const waiting = host.registry(Sum, context); + await host.load([{ name: "@app/sum", facet: defineFacet((h) => h.serve(Sum, { compose: total })) }], context); + expect((await waiting).value).toEqual({ total: 0 }); + await host.close(context); + }); + + it("host.registry waits through no value until a first good composition, woken by it, its wait then gone", async () => { + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total === 0) throw new Error("empty"); + return sum; + }, + }); + setter(host, 0); + }, + }); + const waiting = host.registry(Sum, context, { readyTimeout: 500 }); + await flushed(); + await (await host.get(Setter, {}, context)).set(5, context); + expect((await waiting).value).toEqual({ total: 5 }); + await host.close(context); + }); + + it("once live, a later throwing `compose` changes nothing for the reader: `.value` keeps the last good one", async () => { + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { + compose: (table) => { + const sum = total(table); + if (sum.total < 0) throw new Error("negative"); + return sum; + }, + }); + serveReader(host); + setter(host, 2); + }, + }); + const reader = await host.get(Reader, {}, context); + const setters = await host.get(Setter, {}, context); + expect(reader.total.value).toBe(2); + await setters.set(-1, context); + expect(reader.total.value).toBe(2); + expect(reader.state.value).toEqual({ kind: "live" }); + await host.close(context); + }); +}); + +describe("readers' memory", () => { + it("an ended reader is collected while the registry lives: its derive and its read leave the value", async () => { + const Reader = local("app.reader", { version: 1, params: { id: int() }, members: { total: int() } }); + const { host } = await loadedPlugins({ + "@app/sum": (host) => { + host.serve(Sum, { compose: total, accepts: { 1: { value: ({ total }) => total } } }); + host.serve(Reader, (_, self) => ({ + total: self.derive([self.registry(Sum), self.registry(SumV1)], ([v, old]) => v.total + old), + })); + setter(host); + }, + }); + const refs = await (async () => { + const reader = await host.get(Reader, { id: 1 }, context); + expect(reader.total.value).toBe(2); + reader.release(); + await new Promise((resolve) => setTimeout(resolve, 0)); + return objects.filter((ref) => ref.deref()?.label === "app.reader(1)"); + })(); + expect(refs.length).toBeGreaterThan(0); + await (await host.get(Setter, {}, context)).set(5, context); + await expectCollected(...refs); + await host.close(context); + }); +}); + +describe("values' memory", () => { + it("drops an older version's mapping once its last reader ended (Sol)", async () => { + const New = defineRegistry("app.version-gc", { version: 2 }); + const Old = defineRegistry("app.version-gc", { + version: 1, + }); + const Reader = local("app.old-reader", { version: 1, params: { id: int() }, members: { n: int() } }); + let mapped: WeakRef | undefined; + const { host } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(New, { + compose: () => 1, + accepts: { + 1: { + value: (n) => { + const value = { n, payload: new Uint8Array(1_000_000) }; + mapped = new WeakRef(value); + return value; + }, + }, + }, + }); + h.serve(Reader, (_, self) => ({ n: self.derive([self.registry(Old)], ([v]) => v.n) })); + }, + }); + const reader = await host.get(Reader, { id: 1 }, context); + expect(reader.n.value).toBe(1); + reader.release(); + await flushed(); + await expectCollected(mapped!); + await host.close(context); + }); + + it("keeps no old composition once its readers stopped waiting and nobody follows it (Opus)", async () => { + const Big = defineRegistry("app.big", { version: 1 }); + let first = true; + const made: WeakRef[] = []; + const Waiting = local("app.waiting", { version: 1, params: {}, members: {} }); + const Counter = local("app.counter", { version: 1, params: {}, members: { set: method(int()) } }); + const { host } = await loadedPlugins({ + "@app/a": (h) => { + h.serve(Big, { + compose: (table) => { + if (first) throw new Error("first"); + const value = { payload: new Array(1000).fill(table.length) }; + made.push(new WeakRef(value)); + return value; + }, + }); + h.serve(Counter, (_, self) => { + const value = self.writable(0); + self.contribute(Big, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + h.serve(Waiting, (_, self) => { + void self.registry(Big); + return {}; + }); + }, + }); + const counter = await host.get(Counter, {}, context); + first = false; + // The waiting reader's composition (pulled in the batch of this set) wakes it. + await counter.set(1, context); + await flushed(); + await counter.set(2, context); + void (await host.registry(Big, context)).value; + await settledHeap(); + // Only the current composition is alive. + expect(made.map((ref) => ref.deref() !== undefined)).toEqual([false, true]); + await host.close(context); + }); + + it("a value an app keeps after `host.close` keeps no closed host, and reads its last composition (Sol, Astra)", async () => { + const Kept = defineRegistry("app.kept", { version: 1 }); + const signal = new AbortController(); + const { weak, value } = await (async () => { + const { host } = await loadedPlugins({ "@app/a": (h) => h.serve(Kept, { compose: () => 7 }) }); + const value = await host.registry(Kept, context); + value.observe({ advance: () => {}, end: () => {} }, withAbortSignal(signal.signal, context)); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(1); + await host.close(context); + // The observation ended with the host: its abort listener went with it. + expect(getEventListeners(signal.signal, "abort")).toHaveLength(0); + return { weak: new WeakRef(host), value }; + })(); + await expectCollected(weak); + expect(value.value).toBe(7); + }); +}); + +describe("observe", () => { + it("calls a synchronous observer inside the batch that composed; it counts as watching", async () => { + const Labels = defineRegistry("app.labels", { version: 1 }); + const Label = local("app.label", { version: 1, params: {}, members: { set: method(str()) } }); + const { host } = await loadedPlugins({ + "@app/labels": (host) => { + host.serve(Labels, { compose: (table) => table.map(({ value }) => value).join(",") }); + host.serve(Label, (_, self) => { + const label = self.writable("a"); + self.contribute(Labels, label); + return { + set: async (text: string) => { + label.set(text); + return null; + }, + }; + }); + }, + }); + const labels = await host.registry(Labels, context); + const calls: (NativeWatchCall | WatchEnd)[] = []; + const { context: cancellable, cancel } = withCancel(context); + const observation = labels.observe( + { advance: (call) => calls.push(call), end: (end) => calls.push(end) }, + cancellable, + ); + expect(observation.value).toBe("a"); + const label = await host.get(Label, {}, context); + await label.set("b", context); + // Already called: synchronously, in the set's batch. + expect(calls).toEqual([{ kind: "value", value: "b" }]); + cancel(); + await label.set("c", context); + expect(calls).toEqual([{ kind: "value", value: "b" }, { kind: "aborted" }]); + const again = labels.observe({ advance: (call) => calls.push(call), end: (end) => calls.push(end) }, context); + again.detach(); + await label.set("d", context); + expect(calls).toHaveLength(2); + labels.observe({ advance: () => {}, end: (end) => calls.push(end) }, context); + await host.close(context); + expect(calls.at(-1)).toEqual({ kind: "ended" }); + }); +}); + +describe("a reload's new value adapter (fix round 1)", () => { + it("uses a new value adapter even when compose returns the same primitive", async () => { + const R = defineRegistry("astra.registry", { version: 2 }); + const Old = defineRegistry("astra.registry", { version: 1 }); + const host = createHost({ onError: () => {} }); + const facet = (factor: number) => + defineFacet((h) => h.serve(R, { compose: () => 7, accepts: { 1: { value: (n) => n * factor } } })); + await host.load([{ name: "@astra/reg", facet: facet(2) }], context); + const old = await host.registry(Old, context); + expect(old.value).toBe(14); + await host.reload([{ name: "@astra/reg", facet: facet(3) }], context); + const actual = old.value; + await host.close(context); + expect(actual).toBe(21); + }); +}); diff --git a/packages/chord/test/next/remote/consumer.test.ts b/packages/chord/test/next/remote/consumer.test.ts index 32e4fb839..5f7005a68 100644 --- a/packages/chord/test/next/remote/consumer.test.ts +++ b/packages/chord/test/next/remote/consumer.test.ts @@ -23,6 +23,8 @@ import { rawParent, rawProvider, rejection, + SERVER_ADDRESS, + Server, serveNotes, testCodec, transported, @@ -287,6 +289,7 @@ forEachCodec(() => { version: Notes.version, fingerprint: Notes.fingerprint, params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, }); const [bound] = (await peer.next("bound")).holds; expect(bound!.variant).toBe(EDITOR.variant); @@ -415,6 +418,7 @@ forEachCodec(() => { const Echo = remote("app.echo", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { methods: { echo: method(map(int()), map(int())) } } }, }); const { client } = await linked((h) => diff --git a/packages/chord/test/next/remote/hold.test.ts b/packages/chord/test/next/remote/hold.test.ts index ab1c07b8a..2250a0a8f 100644 --- a/packages/chord/test/next/remote/hold.test.ts +++ b/packages/chord/test/next/remote/hold.test.ts @@ -2,27 +2,40 @@ import { describe, expect, it, vi } from "vitest"; import { type Context, BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import type * as Delta from "../../../src/delta/index.ts"; import { decoder } from "../../../src/delta/index.ts"; -import { bool, int, local, method, obj, remote } from "../../../src/next/contracts/index.ts"; -import { broken } from "../../../src/next/data/index.ts"; +import { bool, evolve, hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { broken, ChordError } from "../../../src/next/data/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; -import { createHost, defineFacet, type Host, type Self } from "../../../src/next/host/index.ts"; +import { createHost, defineFacet, type FacetHost, type Host, type Self } from "../../../src/next/host/index.ts"; import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; import { type Link, memoryLinkPair } from "../../../src/next/link/index.ts"; -import { jsonPart } from "../../../src/next/link/internal.ts"; +import { binaryCodec, jsonPart } from "../../../src/next/link/internal.ts"; import { Hold } from "../../../src/next/remote/hold.ts"; -import { constant } from "../../../src/next/state/index.ts"; +import { constant, type Writable } from "../../../src/next/state/index.ts"; import { changesOf, + delivered, expectCollected, + failLive, fakeTimers, + flushed, forEachCodec, instancesOf, linked, + PlacedNotes, + partValue, RawPeer, rawLinked, + rejection, + relayed, + SERVER_ADDRESS, + Server, + serveNotes, + statesOf, type TestPrincipal, testCodec, transported, + valuesOf, wireNames, } from "../support.ts"; @@ -50,6 +63,7 @@ const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: obj({ n: int() }), half: int() }, methods: { set: method(int()), wait: method() } } }, }); @@ -91,18 +105,26 @@ async function halves() { function hold(peer: RawPeer, id: number, Type: { name: string; version: number; fingerprint: string }) { const { name: type, version, fingerprint } = Type; - peer.send({ kind: "hold", hold: id, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: id, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); } /** The provider object behind a hold, reached through the host's own lookup (to end it by hand). */ function providerObject(server: unknown) { const host = server as ChordHost; const { name: type, version, fingerprint } = Halves; - return host.serving({ type, version, fingerprint, params: "{}" }, {}, { kind: "owner" }, context); + const request = { type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }; + const served = host.serving(request, {}, { kind: "owner" }, context); + if (served.kind !== "object") throw new Error("Halves is served here"); + return served.object; } /** A remote type whose activation never completes: its open waits for its abort. */ -const Pending = remote("app.pending", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); +const Pending = remote("app.pending", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); function servePending(h: Parameters[0]>[0]) { h.serve(Pending, (_, _principal, self) => { @@ -221,7 +243,12 @@ forEachCodec(() => { fakeTimers(); it("a new hold of an object failed transiently is sent the failure at once", async () => { - const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { peer } = await rawLinked((h) => h.serve(Flaky, (_, _principal, self) => { self.open( @@ -309,6 +336,7 @@ forEachCodec(() => { const Slow = local("app.slow", { version: 1, params: {}, members: { n: int() } }); const { server, refs } = await (async () => { const server = createHost({ + address: Server(), principal: { kind: "server" }, readyTimeout: 2_000, onError: () => {}, @@ -343,7 +371,12 @@ forEachCodec(() => { }); /** A remote type whose open throws `offline` (broken when `failing.broken`) while `failing.left` is above 0. */ -const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); +const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); function serveFlaky(h: Parameters[0]>[0], failing: { left: number; broken: boolean }) { h.serve(Flaky, (_, _principal, self) => { @@ -405,7 +438,7 @@ forEachCodec(() => { cause: { kind: "broken", error: { kind: "chord", error: offline } }, }; expect([await peer.next("failed"), await other.next("failed")]).toEqual([broken, broken]); - peer.send({ kind: "retry", hold: 0 }); + peer.send({ kind: "retry", hold: 0, hops: 16 }); for (const consumer of [peer, other]) { expect(await consumer.next("loading")).toEqual({ kind: "loading", hold: 0 }); expect((await consumer.next("bound")).holds[0]).toMatchObject({ hold: 0 }); @@ -419,7 +452,7 @@ forEachCodec(() => { await peer.next("failed"); hold(other, 0, Flaky); await other.next("failed"); - other.send({ kind: "retry", hold: 0 }); + other.send({ kind: "retry", hold: 0, hops: 16 }); for (const consumer of [peer, other]) { await consumer.next("loading"); await consumer.next("bound"); @@ -430,22 +463,83 @@ forEachCodec(() => { const { peer } = await rawLinked((h) => serveFlaky(h, { left: 0, broken: false })); hold(peer, 0, Flaky); await peer.next("bound"); - peer.send({ kind: "retry", hold: 0 }); - peer.send({ kind: "retry", hold: 9 }); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + peer.send({ kind: "retry", hold: 9, hops: 16 }); await transported(); expect([peer.kinds(), peer.closed]).toEqual([["hello"], false]); }); }); + describe("holds that bind to nothing yet (dependency model §1, §3)", () => { + const Absent = { name: "app.absent", version: 1, fingerprint: "0123456789abcdef" }; + + it("a refused hold waits failed { waiting }, counted; a retry re-runs its start (loading, failed); its link's close drops it", async () => { + const { server, peer } = await rawLinked(() => {}); + hold(peer, 0, Absent); + const failed = await peer.next("failed"); + expect(failed).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + expect(failed.retryIn).toBeUndefined(); + expect(server.activity.value.references).toBe(1); + expect(server.references(Halves).value).toEqual([]); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + expect(await peer.next("loading")).toEqual({ kind: "loading", hold: 0 }); + expect(await peer.next("failed")).toMatchObject({ hold: 0, cause: { kind: "waiting" } }); + peer.close(); + await transported(); + expect(server.activity.value.references).toBe(0); + }); + + it("a hold on an object parked on a failed dependency is told each new cause, then loading, then bound", async () => { + const Core = local("app.core", { version: 1, params: {}, members: { n: int() } }); + let cores = 0; + const { peer } = await rawLinked((h) => { + h.serve(Core, () => { + if (++cores < 3) throw new Error(`bug ${cores}`); + return { n: constant(cores) }; + }); + h.serve(Halves, (_, _principal, self) => { + const core = self.get(Core, {}); + const n = self.state(N); + return { + variant: "v", + members: { + n, + half: self.derive([core.n], ([c]) => c), + set: async () => null, + wait: async () => null, + }, + }; + }); + }); + hold(peer, 0, Halves); + const waiting = (message: string) => ({ + kind: "failed", + hold: 0, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "unavailable", message } } }, + }); + // The core failed at load (its first outcome): the parked object names it at once. + expect(await peer.next("failed")).toEqual(waiting("waiting for app.core: app.core: bug 1")); + // A retry passes on through the parked object to its broken dependency (one host's retry operation). + // (Its new attempt failed within the batch of changes the parked object checks once: no loading between.) + peer.send({ kind: "retry", hold: 0, hops: 16 }); + expect(await peer.next("failed")).toEqual(waiting("waiting for app.core: app.core: bug 2")); + peer.send({ kind: "retry", hold: 0, hops: 16 }); + await peer.next("loading"); + expect((await peer.next("bound")).holds[0]).toMatchObject({ hold: 0 }); + }); + }); + describe("a reconnecting consumer", () => { const Tally = remote("app.tally", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, }); const Fresh = remote("app.fresh", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: obj({ n: int() }) }, methods: { bump: method() } } }, }); @@ -521,6 +615,7 @@ forEachCodec(() => { const Retiring = remote("app.retiring", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { doc: obj({ n: int() }), big: bool(), t: int() }, @@ -536,7 +631,7 @@ const retiredAll = [RETIRING.state.doc!, RETIRING.state.big!, RETIRING.state.t!] /** A server serving `Retiring` to a raw peer that acks only when told; `call` calls a method of hold 0. */ async function retiring() { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( [ { @@ -583,7 +678,7 @@ async function retiring() { expect((await peer.next("result")).call).toBe(id); }; const { name: type, version, fingerprint } = Retiring; - peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}" }); + peer.send({ kind: "hold", hold: 0, type, version, fingerprint, params: "{}", address: SERVER_ADDRESS }); await peer.next("bound"); peer.ack(); return { peer, call }; @@ -692,3 +787,583 @@ forEachCodec(() => { }); }); }); + +/** Lets every pending reaction run (a memory link delivers by microtask). */ +async function flush(): Promise { + for (let i = 0; i < 100; i++) await Promise.resolve(); +} + +/** A local type nobody serves. */ +const Missing = local("app.missing", { version: 1, params: {}, members: { n: int() } }); + +describe("retries across links: the hop budget (audit A5)", () => { + it.each([1, 9, 50])( + "quiesce after %i concurrent Retries round a waiting ring across hosts: each stops at its budget, no attempt started", + async (operations) => { + type P = { kind: "server" } | { kind: "worker" }; + const Server = hostKind("app.retry-server"); + const Worker = hostKind("app.retry-worker"); + const LocalA = local("app.a", { version: 1, params: {}, members: { n: int() } }); + const LocalB = local("app.b", { version: 1, params: {}, members: { n: int() } }); + const RemoteA = remote("app.remote-a", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { reader: { state: { n: int() } } }, + }); + const RemoteB = remote("app.remote-b", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { reader: { state: { n: int() } } }, + }); + const hops: number[] = []; + let factories = 0; + const count = (link: Link): Link => ({ + send: (bytes) => { + const message = binaryCodec.decode(bytes); + if (message.kind === "retry") hops.push(message.hops); + link.send(bytes); + }, + onMessage: (listener) => link.onMessage(listener), + onClose: (listener) => link.onClose(listener), + close: () => link.close(), + }); + const [down, up] = memoryLinkPair(); + const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); + server.route( + Worker, + { + authorize: () => true, + resolve: async () => ({ link: count(down), principal: { kind: "worker" }, trusted: true }), + }, + context, + ); + worker.connect(count(up), { principal: { kind: "server" }, trusted: true }, context); + try { + const left = server.load( + [ + { + name: "@test/a", + facet: defineFacet

((h) => { + h.serve(LocalA, (_, self) => { + factories++; + return { n: self.get(RemoteB, {}).n }; + }); + h.serve(RemoteA, (_, _principal, self) => { + factories++; + return { variant: "reader", members: { n: self.get(LocalA, {}).n } }; + }); + }), + }, + ], + context, + ); + const right = worker.load( + [ + { + name: "@test/b", + facet: defineFacet

((h) => { + h.serve(LocalB, (_, self) => { + factories++; + self.get(Missing, {}); + return { n: self.get(RemoteA, {}).n }; + }); + h.serve(RemoteB, (_, _principal, self) => { + factories++; + return { variant: "reader", members: { n: self.get(LocalB, {}).n } }; + }); + }), + }, + ], + context, + ); + await Promise.all([left, right]); + await flush(); + const a = server.hold(LocalA, {}, context); + expect(a.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + const made = factories; + for (let i = 0; i < operations; i++) a.retry(); + await flush(); + // Each passed round the ring with one hop less per host (16 from the first), never sent at 0. + const lap = Array.from({ length: 16 }, (_, i) => 16 - i); + expect(hops.toSorted((x, y) => y - x)).toEqual(lap.flatMap((hop) => Array(operations).fill(hop))); + await flush(); + expect(hops).toHaveLength(16 * operations); + // Parked objects only passed it on: no attempt started. + expect(factories).toBe(made); + } finally { + await server.close(context); + await worker.close(context); + } + }, + ); +}); + +/** A remote object on the server: a label, a counter and `bump`. */ +const Shown = remote("app.shown", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { label: str(), n: int() }, methods: { bump: method(undefined, int()) } } }, +}); + +/** `Shown` v2: its label became an object; v1 consumers map back through `accepts`. */ +const ShownV2 = evolve(Shown, (e) => e.state("v", "label", obj({ text: str() }))); + +/** Serves `Shown` labelled `label` (with a native when `native`); `bump` adds 1 to its in-memory counter. */ +function serveShown(h: FacetHost, label: string, native = false): void { + h.serve(Shown, (_, _principal, self) => { + const doc = self.state(N); + if (native) + self.open( + async () => label, + () => {}, + ); + return { + variant: "v", + members: { + label: constant(label), + n: self.derive([doc], ([d]) => d.n), + bump: async (ctx) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }, + }; + }); +} + +/** Serves `ShownV2` labelled `text`; with `accepts`, v1 consumers see `label` as its text. */ +function serveShownV2(h: FacetHost, text: string, accept: boolean): void { + h.serve( + ShownV2, + (_, _principal, self) => { + const doc = self.state(N); + return { + variant: "v", + members: { + label: constant({ text }), + n: self.derive([doc], ([d]) => d.n), + bump: async (ctx) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }, + }; + }, + accept ? { accepts: { 1: { state: { v: { label: (current) => current.text } } } } } : {}, + ); +} + +/** Reloads the server's plugin `@test/app` (as `linked` names it) with `setup`. */ +function reloadServer(server: Host, setup: (host: FacetHost) => void) { + return server.reload([{ name: "@test/app", facet: defineFacet(setup) }], context); +} + +describe("holds across a reload (slice 7b §6)", () => { + it("a swapped object keeps its holds: a fresh bound once the new instance is live, never loading (audit A9)", async () => { + const { server, client } = await linked((h) => serveShown(h, "v1")); + const shown = await client.get(Shown, {}, context); + await shown.bump(context); + const states = statesOf(await shown.state.watch(context)); + const label = changesOf(await shown.label.watch(context)); + const n = changesOf(await shown.n.watch(context)); + await reloadServer(server, (h) => serveShown(h, "v2")); + await transported(); + expect(label).toEqual([{ kind: "value", value: "v2", ops: [["r", "v2"]], retired: false }]); + // The fresh bound carries every member: an unchanged one may show one frame (nothing lost or stale). + expect(valuesOf(n)).toEqual([1]); + expect(states).toEqual([]); + expect(await shown.bump(context)).toBe(2); + await server.close(context); + }); + + it("a restarted object's holds see loading, then a fresh bound", async () => { + const { server, client } = await linked((h) => serveShown(h, "v1", true)); + const shown = await client.get(Shown, {}, context); + const states = statesOf(await shown.state.watch(context)); + const label = changesOf(await shown.label.watch(context)); + await reloadServer(server, (h) => serveShown(h, "v2", true)); + await transported(); + expect(states).toEqual(["loading", "live"]); + expect(valuesOf(label)).toEqual(["v2"]); + await server.close(context); + }); + + it("a version the new code no longer accepts unbinds its holds (version); a later reload accepting it binds them", async () => { + const { server, client } = await linked((h) => serveShown(h, "v1")); + const shown = await client.get(Shown, {}, context); + await reloadServer(server, (h) => serveShownV2(h, "v2", false)); + await transported(); + expect(shown.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, + }); + expect(await rejection(shown.bump(context))).toMatchObject({ code: "contract_mismatch" }); + await reloadServer(server, (h) => serveShownV2(h, "v3", true)); + await transported(); + expect(shown.state.value).toEqual({ kind: "live" }); + expect(shown.label.value).toBe("v3"); + await server.close(context); + }); + + it("a removed type unbinds its holds (not_served); served again, they bind and a derive over them heals", async () => { + const Panel = local("app.panel", { version: 1, params: {}, members: { text: str() } }); + const { server, client } = await linked((h) => serveShown(h, "v1")); + await client.reload( + [ + { + name: "@test/panel", + facet: defineFacet((h) => + h.serve(Panel, (_, self) => { + const shown = self.get(Shown, {}); + return { text: self.derive([shown.label], ([label]) => `panel ${label}`) }; + }), + ), + }, + ], + context, + ); + const panel = await client.get(Panel, {}, context); + const shown = client.hold(Shown, {}, context); + await server.reload([], context); + await transported(); + expect(shown.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { code: "unavailable" } }, + }); + // The dependent stays live (R18) with its last value. + expect(panel.state.value).toEqual({ kind: "live" }); + expect(panel.text.value).toBe("panel v1"); + await server.reload( + [{ name: "@test/app", facet: defineFacet((h) => serveShown(h, "v2")) }], + context, + ); + await transported(); + expect(shown.state.value).toEqual({ kind: "live" }); + expect(panel.text.value).toBe("panel v2"); + await server.close(context); + }); + + it("a hold unbound while the window is full drops its pending ops; acks of messages sent stay counted", async () => { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => serveShown(h, "v1")) }], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + const SHOWN = wireNames(Shown, "v"); + await peer.next("hello"); + hold(peer, 0, Shown); + await peer.next("bound"); + for (let call = 0; call < 3; call++) { + peer.send({ kind: "call", call, hold: 0, method: SHOWN.methods.bump! }); + await peer.next("result"); + } + // One state in flight with the bound: the third commit's ops wait in their pending queue. + await peer.next("state"); + await server.reload([], context); + await transported(); + expect((await peer.next("failed")).cause).toMatchObject({ kind: "waiting" }); + peer.ack(); + peer.ack(); + await transported(); + expect(peer.kinds()).toEqual([]); + await server.close(context); + }); + + it("a swap sends each bound hold one fresh bound with the new instance's values, no state (audit A9)", async () => { + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load( + [{ name: "@test/app", facet: defineFacet((h) => serveShown(h, "v1")) }], + context, + ); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b); + const SHOWN = wireNames(Shown, "v"); + await peer.next("hello"); + hold(peer, 0, Shown); + hold(peer, 1, Shown); + await peer.next("bound"); + await transported(); + peer.received.length = 0; + await reloadServer(server, (h) => serveShown(h, "v2")); + await transported(); + expect(new Set(peer.kinds())).toEqual(new Set(["bound"])); + const entries = peer.received.flatMap((message) => (message.kind === "bound" ? message.holds : [])); + expect(entries.map(({ hold }) => hold).sort()).toEqual([0, 1]); + for (const entry of entries) expect(SHOWN.named(entry.members)).toEqual({ label: "v2", n: 0 }); + await server.close(context); + }); + + it("a swap during a full window: calls are answered by the old names until the fresh bound goes (audit A9)", async () => { + const Two = remote("app.two-variants", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { + a: { state: { label: str() }, methods: { bump: method(undefined, int()), ping: method(undefined, int()) } }, + b: { state: { label: str() }, methods: { ping: method(undefined, int()) } }, + }, + }); + const serveTwo = (variant: "a" | "b", label: string) => + defineFacet((h) => + h.serve(Two, (_, _principal, self) => { + const doc = self.state(N); + const ping = async () => 7; + const bump = async (ctx: Context) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }; + return variant === "a" + ? { variant, members: { label: constant(label), bump, ping } } + : { variant, members: { label: constant(label), ping } }; + }), + ); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@test/app", facet: serveTwo("a", "v1") }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + const A = wireNames(Two, "a"); + await peer.next("hello"); + hold(peer, 0, Two); + await peer.next("bound"); + // A second message in flight: the window (two) is full. + const changed = defineFacet((h) => + h.serve(Two, (_, _principal, self) => ({ + variant: "a", + members: { + label: self.derive([self.state(N)], ([d]) => `n ${d.n}`), + bump: async () => 0, + ping: async () => 7, + }, + })), + ); + await server.reload([{ name: "@test/app", facet: changed }], context); + await peer.next("bound"); + // Swaps in place while the window is full: the fresh bound waits for a slot, the hold answers by its old names. + await server.reload([{ name: "@test/app", facet: serveTwo("a", "v2") }], context); + await transported(); + peer.send({ kind: "call", call: 0, hold: 0, method: A.methods.bump! }); + expect(partValue((await peer.next("result")).value!)).toBe(1); + // The new variant lacks a method of the old names: the attempt refuses it. + await server.reload([{ name: "@test/app", facet: serveTwo("b", "v3") }], context); + await transported(); + peer.send({ kind: "call", call: 1, hold: 0, method: A.methods.bump! }); + expect(await peer.next("error")).toMatchObject({ error: { error: { code: "invalid", reason: "member" } } }); + peer.send({ kind: "call", call: 2, hold: 0, method: A.methods.ping! }); + expect(partValue((await peer.next("result")).value!)).toBe(7); + expect(peer.kinds().filter((kind) => kind === "bound")).toEqual([]); + peer.ack(); + const fresh = await peer.next("bound"); + expect(fresh.holds[0]).toMatchObject({ hold: 0, variant: wireNames(Two, "b").variant }); + expect(wireNames(Two, "b").named(fresh.holds[0]!.members)).toEqual({ label: "v3" }); + await server.close(context); + }); + + it("collects the swapped-out instance while the hold's fresh bound waits for a slot (Astra, audit A9 GC)", async () => { + const Type = remote("astra.audit-gc", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { n: int() } } }, + }); + let declared: Self | undefined; + const facet = (n: number) => + defineFacet((h) => + h.serve(Type, (_, _principal, self) => { + declared = self; + return { variant: "main", members: { n: constant(n) } }; + }), + ); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + await server.load([{ name: "@astra/audit-gc", facet: facet(1) }], context); + const [a, b] = memoryLinkPair(); + server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); + const peer = new RawPeer(b, { ack: false }); + await peer.next("hello"); + hold(peer, 0, Type); + await peer.next("bound"); + // A second bound in flight: the window is full. + await server.reload([{ name: "@astra/audit-gc", facet: facet(2) }], context); + await peer.next("bound"); + const retired = new WeakRef(declared!); + declared = undefined; + await server.reload([{ name: "@astra/audit-gc", facet: facet(3) }], context); + await transported(); + await flushed(); + // Its fresh bound waits for an ack; the hold keeps the old names for calls, not the old instance. + expect(peer.kinds().filter((kind) => kind === "bound")).toEqual([]); + await expectCollected(retired); + peer.ack(); + await peer.next("bound"); + await server.close(context); + }); + + it("through a relay: a worker's reload reaches the tab as it reaches a direct consumer", async () => { + const { tab, workers, server } = await relayed(); + const notes = await tab.get(PlacedNotes, { session: "S1" }, context); + const worker = workers.get("session|S1")!; + const states = statesOf(await notes.state.watch(context)); + await worker.reload([], context); + await transported(); + expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await worker.reload( + [{ name: "@test/worker", facet: defineFacet((h) => void serveNotes(h, PlacedNotes)) }], + context, + ); + await transported(); + expect(notes.state.value).toEqual({ kind: "live" }); + expect(states.at(-1)).toBe("live"); + await server.close(context); + }); +}); + +describe("holds across more than one swap (fix round 1)", () => { + it("keeps a real subscriber when a changed constant later becomes dynamic", async () => { + type P = { kind: "server" } | { kind: "tab" }; + const Server = hostKind("sol-server"); + const Type = remote("sol.wire", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { x: int() }, methods: { set: method(int()) } } }, + }); + const Doc = defineState("wire", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + const facet = (choice: { kind: "constant"; n: number } | { kind: "dynamic" }) => + defineFacet

((h) => + h.serve(Type, (_, _principal, self) => { + const doc = self.state(Doc); + return { + variant: "main", + members: { + x: choice.kind === "constant" ? constant(choice.n) : self.derive([doc], ([value]) => value.n), + set: async (n: number, ctx: Context) => { + await doc.change((draft) => { + draft.n = n; + }, ctx); + return null; + }, + }, + }; + }), + ); + const server = createHost

({ address: Server(), principal: { kind: "server" }, onError: () => {} }); + const tab = createHost

({ onError: () => {} }); + try { + await server.load([{ name: "@sol/wire", facet: facet({ kind: "constant", n: 1 }) }], context); + await tab.load([], context); + const [up, down] = memoryLinkPair(); + server.accept(up, { principal: { kind: "tab" } }, context); + tab.connect(down, { principal: { kind: "server" } }, context); + const held = await tab.get(Type, {}, context); + await server.reload([{ name: "@sol/wire", facet: facet({ kind: "constant", n: 2 }) }], context); + await delivered(); + expect(held.x.value).toBe(2); + await server.reload([{ name: "@sol/wire", facet: facet({ kind: "dynamic" }) }], context); + await delivered(); + expect(held.x.value).toBe(0); + await held.set(7, context); + await delivered(); + expect(held.x.value).toBe(7); + } finally { + await tab.close(context); + await server.close(context); + } + }); +}); + +describe("a hold after its object's live failure (final review)", () => { + const attempted = instancesOf(ChordObject, "attempt"); + + it("follows a member re-exposing another object's source until the admitted commits published (audit A10)", async () => { + const Point = obj({ n: int() }); + const A = local("app.foreign", { version: 1, params: {}, members: { x: Point } }); + const B = remote("app.reexposing", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { x: Point } } }, + }); + let foreign: Writable<{ n: number }> | undefined; + let self: Self | undefined; + const { server, client } = await linked((h) => { + h.serve(A, (_, s) => { + foreign = s.writable({ n: 0 }); + return { x: s.derive([foreign], ([value]) => value) }; + }); + h.serve(B, (_, _principal, s) => { + self = s; + return { variant: "main", members: { x: s.get(A, {}).x } }; + }); + }); + const ref = await client.get(B, {}, context); + const calls = changesOf(await ref.x.watch(context)); + failLive(attempted, self!, new ChordError({ code: "unavailable", message: "boom" })); + // A's change lands after B failed, in the window before B's admitted commits published: it shows (one more + // change), then the member holds with the failure. + foreign!.set({ n: 1 }); + await delivered(); + expect([ref.x.value, valuesOf(calls)]).toEqual([{ n: 1 }, [{ n: 1 }, { failed: "boom" }]]); + await client.close(context); + await server.close(context); + }); +}); + +describe("a hold across a swap whose mapping fails (final review)", () => { + it("gets the member's failure in its fresh bound and heals with the new source (Sol, audit A9)", async () => { + const W1 = remote("app.swap-failed", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { main: { state: { x: int() } } }, + }); + const W2 = evolve(W1, (e) => e.state("main", "x", obj({ n: int() }))); + let throws = true; + let point: Writable<{ n: number }> | undefined; + const serve = (h: FacetHost, map: (value: { n: number }) => number) => + h.serve( + W2, + (_, _principal, s) => { + point = s.writable({ n: 1 }); + return { variant: "main", members: { x: s.derive([point], ([value]) => value) } }; + }, + { accepts: { 1: { state: { main: { x: map } } } } }, + ); + const { server, client } = await linked((h) => serve(h, (value) => value.n)); + const ref = await client.get(W1, {}, context); + const calls = changesOf(await ref.x.watch(context)); + // The new instance's mapping throws for its first value. + const failing = (value: { n: number }) => { + if (throws) throw new Error("member boom"); + return value.n; + }; + await server.reload( + [{ name: "@test/app", facet: defineFacet((h) => serve(h, failing)) }], + context, + ); + await delivered(); + // Live, the member failed: its watch got the failure, a read throws it (as a failed member does on the host). + expect([ref.state.value.kind, valuesOf(calls)]).toEqual(["live", [{ failed: "member boom" }]]); + expect(() => ref.x.value).toThrow("member boom"); + // It follows the new source: the next value heals it. + throws = false; + point!.set({ n: 3 }); + await delivered(); + expect([ref.x.value, valuesOf(calls)]).toEqual([3, [{ failed: "member boom" }, 3]]); + await client.close(context); + await server.close(context); + }); +}); diff --git a/packages/chord/test/next/remote/host-link.test.ts b/packages/chord/test/next/remote/host-link.test.ts index 7eb0a8551..874287405 100644 --- a/packages/chord/test/next/remote/host-link.test.ts +++ b/packages/chord/test/next/remote/host-link.test.ts @@ -3,7 +3,7 @@ import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; import { createHost, defineFacet, type FacetHost, type HostError } from "../../../src/next/host/index.ts"; import { type Compression, memoryLinkPair, type PartLimits } from "../../../src/next/link/index.ts"; -import { jsonPart } from "../../../src/next/link/internal.ts"; +import { jsonPart, type Message } from "../../../src/next/link/internal.ts"; import { constant } from "../../../src/next/state/index.ts"; import { nodeCompression } from "../../../src/next-node/compression.ts"; import { @@ -19,11 +19,15 @@ import { partValue, RAW_PEER, RawPeer, + rawLinked, rawWorkers, rejection, relayed, + SERVER_ADDRESS, + Server, Session, serveNotes, + settledHeap, type TestPrincipal, testCodec, transported, @@ -33,6 +37,7 @@ import { const ServerWho = remote("app.server-who", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { who: str() }, methods: { echo: method(str(), str()) } } }, }); @@ -40,7 +45,7 @@ const ServerWho = remote("app.server-who", { const Asker = remote("app.asker", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { v: { state: { who: str() }, methods: { ask: method(str(), str()) } } }, }); @@ -48,7 +53,7 @@ const Asker = remote("app.asker", { const Neighbour = remote("app.neighbour", { version: 1, params: { session: str(), other: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: { v: { state: { count: int() } } }, }); @@ -141,6 +146,7 @@ forEachCodec(() => { const Gateway = remote("app.gateway", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { n: int() }, methods: { open: method(str(), int()) } } }, }); // With params: not activated by `load`, which would wait for the raw worker. @@ -342,6 +348,7 @@ async function compressing(options: { }) { const errors: HostError[] = []; const server = createHost({ + address: Server(), principal: { kind: "server" }, compression: options.server, onError: (error) => errors.push(error), @@ -427,3 +434,76 @@ forEachCodec(() => { }); }); }); + +describe("origins only from relays (decision, slice 10 security audit K1/K2)", () => { + const { name: type, version, fingerprint } = Notes; + const hold = { + kind: "hold", + hold: 0, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; + const withOrigin: Record = { + hold: { ...hold, origin: 1 } as Message, + call: { kind: "call", origin: 1, call: 0, hold: 0, method: EDITOR.methods.wait }, + release: { kind: "release", origin: 1, hold: 0 }, + abort: { kind: "abort", origin: 1, call: 0 }, + retry: { kind: "retry", origin: 1, hold: 0, hops: 1 } as Message, + ack: { kind: "ack", origin: 1 }, + closed: { kind: "closed", origin: 1 }, + }; + + for (const [kind, message] of Object.entries(withOrigin)) { + it(`closes a child link sending ${kind} with an origin (a protocol error)`, async () => { + const { peer, errors } = await rawLinked((h) => void serveNotes(h)); + peer.send(message); + await transported(); + expect(peer.closed).toBe(true); + expect(errors).toMatchObject([ + { kind: "protocol_error", error: { message: `${kind}: an origin on a child link` } }, + ]); + }); + } + + it("honours origins on a parent link (a relay's): one window of calls per origin", async () => { + let probe!: ReturnType; + const { peer } = await rawLinked( + (h) => { + probe = serveNotes(h); + }, + { role: "parent" }, + ); + peer.send({ ...hold, origin: 1 } as Message); + peer.send({ ...hold, origin: 2 } as Message); + await transported(); + for (const origin of [1, 2]) peer.send({ kind: "call", origin, call: 0, hold: 0, method: EDITOR.methods.wait }); + await transported(); + expect([peer.closed, probe.waits.length]).toEqual([false, 2]); + }); + + it("keeps no record of an origin that holds nothing: 20,000 fresh origins held and released", async () => { + const { peer } = await rawLinked((h) => void serveNotes(h), { role: "parent" }); + const churn = async (from: number, to: number) => { + for (let origin = from; origin <= to; origin++) { + peer.send({ ...hold, origin } as Message); + peer.send({ kind: "release", origin, hold: 0 }); + if (origin % 1_000 === 0) { + await transported(); + peer.received.length = 0; + } + } + await transported(); + peer.received.length = 0; + }; + // A warm-up first: what a first run allocates for good (compiled code, caches) is no record. + await churn(1, 2_000); + const before = await settledHeap(); + await churn(2_001, 22_000); + // About 1 KiB per origin before: 20 MiB. The bound leaves room for heap noise (CI measured 0.6 MiB). + expect((await settledHeap()) - before).toBeLessThan(4 * 1024 * 1024); + expect(peer.closed).toBe(false); + }, 60_000); +}); diff --git a/packages/chord/test/next/remote/path.test.ts b/packages/chord/test/next/remote/path.test.ts index c3efb197d..ff58b786c 100644 --- a/packages/chord/test/next/remote/path.test.ts +++ b/packages/chord/test/next/remote/path.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "vitest"; +import { ChordError } from "../../../src/next/data/index.ts"; import type { HostLink } from "../../../src/next/remote/host-link.ts"; import { Path, type Upstream } from "../../../src/next/remote/path.ts"; @@ -67,4 +68,23 @@ describe("a path", () => { path.connected(linkOf("2")); expect(log).toEqual(["a@1", "b@1"]); }); + + it("says what changed: a holder attached without a link, a drop, the last release; fails its holders waiting", () => { + const events: string[] = []; + const failed: string[] = []; + const path = new Path((event) => events.push(event.kind)); + const a = { + attach: () => {}, + dropped: () => {}, + unreachable: (error: { readonly message: string }) => void failed.push(error.message), + } as unknown as Upstream; + path.attach(a); + path.unreachable(new ChordError({ code: "unavailable", message: "no worker" })); + const link = linkOf("1"); + path.connected(link); + path.disconnected(link); + path.detach(a); + expect(events).toEqual(["attached", "dropped", "released"]); + expect(failed).toEqual(["no worker"]); + }); }); diff --git a/packages/chord/test/next/remote/provider.test.ts b/packages/chord/test/next/remote/provider.test.ts index b1be51b9e..5abb539e3 100644 --- a/packages/chord/test/next/remote/provider.test.ts +++ b/packages/chord/test/next/remote/provider.test.ts @@ -21,6 +21,8 @@ import { RAW_PEER, RawPeer, rawLinked, + SERVER_ADDRESS, + Server, serveNotes, type TestPrincipal, testCodec, @@ -39,7 +41,16 @@ function hold( ) { const { name: type, version, fingerprint } = { ...Type, ...override }; const from = origin === undefined ? {} : { origin }; - peer.send({ kind: "hold", ...from, hold: id, type, version, fingerprint, params: jsonPart(params) }); + peer.send({ + kind: "hold", + ...from, + hold: id, + type, + version, + fingerprint, + params: jsonPart(params), + address: SERVER_ADDRESS, + }); } /** Sends a call of the method at index `method` of the bound variant, with `args` (none when left out). */ @@ -50,7 +61,12 @@ function call(peer: RawPeer, id: number, holdId: number, method: number, args?: } /** A remote type kept per id. */ -const Kept = remote("app.kept", { version: 1, params: { id: int() }, variants: { v: { state: { n: int() } } } }); +const Kept = remote("app.kept", { + version: 1, + params: { id: int() }, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, +}); /** The wire's indexes of `Notes`'s reader variant. */ const READER = wireNames(Notes, "reader"); @@ -58,13 +74,13 @@ const READER = wireNames(Notes, "reader"); const { add, fail, wait } = EDITOR.methods as { add: number; fail: number; wait: number }; /** A server serving `Notes` and a raw consumer peer stamped `child`. */ -async function notesServer(child: TestPrincipal = { kind: "owner" }) { +async function notesServer(child: TestPrincipal = { kind: "owner" }, role: "child" | "parent" = "child") { let probe: NotesProbe | undefined; const linked = await rawLinked( (h) => { probe = serveNotes(h); }, - { child }, + { child, role }, ); return { ...linked, probe: probe! }; } @@ -101,14 +117,14 @@ forEachCodec(() => { { fingerprint: "0000000000000000" }, { code: "contract_mismatch", message: "app.notes v1 does not match the served v1" }, ], - ])("refuse %s with a broken failed", async (_, override, error) => { + ])("keep %s refused, failed { waiting }", async (_, override, error) => { const { peer } = await notesServer(); const { type, ...rest } = override as { type?: string }; hold(peer, 0, { ...Notes, name: type ?? Notes.name } as Contract, { session: "S1" }, rest); - expect(await peer.next("failed")).toMatchObject({ - hold: 0, - cause: { kind: "broken", error: { kind: "chord", error } }, - }); + const failed = await peer.next("failed"); + expect(failed).toMatchObject({ hold: 0, cause: { kind: "waiting", error: { kind: "chord", error } } }); + // No backoff window: an event (a publication) wakes it. + expect(failed.retryIn).toBeUndefined(); }); it("refuse invalid params with a broken failed", async () => { @@ -119,8 +135,12 @@ forEachCodec(() => { }); }); - it("arriving before load resolved wait for it", async () => { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + it("arriving before the publication wait for it, and bind before the start barrier is over", async () => { + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); const [a, b] = memoryLinkPair(); server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); @@ -177,7 +197,12 @@ forEachCodec(() => { }); it("before bound are refused not_ready (race 3)", async () => { - const Slow = remote("app.slow", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Slow = remote("app.slow", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { peer } = await rawLinked((h) => h.serve(Slow, (_, _principal, self) => { self.open( @@ -333,7 +358,11 @@ forEachCodec(() => { it("a hold released while it waits for load takes nothing once load resolves", async () => { const events: string[] = []; - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); const [a, b] = memoryLinkPair(); server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); const peer = new RawPeer(b); @@ -354,6 +383,7 @@ forEachCodec(() => { const Slow = remote("app.slow", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); const aborted: string[] = []; @@ -381,6 +411,7 @@ forEachCodec(() => { const Closing = remote("app.closing", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); const events: string[] = []; @@ -429,6 +460,7 @@ forEachCodec(() => { const Counter = remote("app.counter", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int(), seen: int() }, methods: { bump: method() } } }, }); const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); @@ -473,12 +505,12 @@ forEachCodec(() => { }); describe("local types over a link", () => { - it("never cross one", async () => { + it("never cross one: a hold naming one is not served (failed { waiting })", async () => { const Store = local("app.store", { version: 1, params: {}, members: { n: int() } }); const { peer } = await rawLinked((h) => h.serve(Store, () => ({ n: constant(1) }))); hold(peer, 0, Store, {}); expect(await peer.next("failed")).toMatchObject({ - cause: { kind: "broken", error: { kind: "chord", error: { code: "unavailable" } } }, + cause: { kind: "waiting", error: { kind: "chord", error: { code: "unavailable" } } }, }); }); }); @@ -488,6 +520,7 @@ forEachCodec(() => { const Grid = remote("app.grid", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { grid: obj({ cells: map(int()) }), one: int() }, @@ -518,9 +551,9 @@ function serveGrid(h: FacetHost): void { } /** A server serving `Notes` and `Grid` to a raw peer that acks only when told, the link's context counting. */ -async function slowConsumer() { +async function slowConsumer(role: "child" | "parent" = "child") { const counting = await metered(); - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + const server = createHost({ address: Server(), principal: { kind: "server" }, onError: () => {} }); await server.load( [ { @@ -534,7 +567,9 @@ async function slowConsumer() { context, ); const [a, b] = memoryLinkPair(); - server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, counting.context); + const options = { principal: { kind: "owner" }, codec: testCodec() } as const; + if (role === "parent") server.connect(a, options, counting.context); + else server.accept(a, options, counting.context); const peer = new RawPeer(b, { ack: false }); await transported(); return { server, peer, counts: counting.counts }; @@ -634,9 +669,9 @@ forEachCodec(() => { }); }); - describe("origins at the provider", () => { + describe("origins at the provider (a relay's: on its parent link)", () => { it("key holds and calls by (origin, id): the same ids on two origins are two holds; answers carry the origin", async () => { - const { peer } = await slowConsumer(); + const { peer } = await slowConsumer("parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S2" }, {}, 3); const [own, relayed] = [await peer.next("bound"), await peer.next("bound")]; @@ -652,7 +687,7 @@ forEachCodec(() => { }); it("keep a window per origin: a stalled origin folds its commits, another keeps receiving each", async () => { - const { server, peer } = await slowConsumer(); + const { server, peer } = await slowConsumer("parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S1" }, {}, 1); await peer.next("bound"); @@ -677,7 +712,7 @@ forEachCodec(() => { }); it("count calls per origin against N: 256 on each of two origins run, one more on one closes the link", async () => { - const { peer, errors } = await notesServer(); + const { peer, errors } = await notesServer({ kind: "owner" }, "parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S1" }, {}, 1); await peer.next("bound"); @@ -697,7 +732,7 @@ forEachCodec(() => { }); it("ignore a call on a hold they do not know before counting it (Q1)", async () => { - const { peer, errors } = await notesServer(); + const { peer, errors } = await notesServer({ kind: "owner" }, "parent"); hold(peer, 0, Notes, { session: "S1" }); await peer.next("bound"); for (let id = 0; id < 256; id++) call(peer, id, 0, wait); @@ -709,7 +744,7 @@ forEachCodec(() => { }); it("closed releases the origin's holds and aborts its calls; another origin keeps its own", async () => { - const { peer, probe, server } = await notesServer(); + const { peer, probe, server } = await notesServer({ kind: "owner" }, "parent"); hold(peer, 0, Notes, { session: "S1" }); hold(peer, 0, Notes, { session: "S2" }, {}, 2); await peer.next("bound"); @@ -729,7 +764,12 @@ forEachCodec(() => { }); it("carry constants and failed members, and an empty variant's binding, in a first bound (race 24)", async () => { - const Empty = remote("app.empty", { version: 1, params: {}, variants: { v: {} } }); + const Empty = remote("app.empty", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: {} }, + }); const N = defineState("n", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); const { peer } = await rawLinked((h) => { h.serve(Empty, () => ({ variant: "v", members: {} })); @@ -786,7 +826,7 @@ forEachCodec(() => { push.call(this, ops); }; try { - const { server, peer } = await slowConsumer(); + const { server, peer } = await slowConsumer("parent"); hold(peer, 0, Notes, { session: "S1" }, {}, 1); await peer.next("bound"); for (const text of ["a", "b", "c"]) await addNote(server, "S1", text); @@ -836,6 +876,7 @@ forEachCodec(() => { const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { one: int(), half: int() }, methods: { set: method(int()) } } }, }); @@ -968,7 +1009,15 @@ forEachCodec(() => { /** A hold of `Notes` for S1 as a message. */ function notesHold(id: number): Extract { const { name: type, version, fingerprint } = Notes; - return { kind: "hold", hold: id, type, version, fingerprint, params: jsonPart({ session: "S1" }) }; + return { + kind: "hold", + hold: id, + type, + version, + fingerprint, + params: jsonPart({ session: "S1" }), + address: SERVER_ADDRESS, + }; } const UTF8 = new TextEncoder(); diff --git a/packages/chord/test/next/remote/relay.test.ts b/packages/chord/test/next/remote/relay.test.ts index 701b2adaf..6f08a6f14 100644 --- a/packages/chord/test/next/remote/relay.test.ts +++ b/packages/chord/test/next/remote/relay.test.ts @@ -21,6 +21,8 @@ import { rawWorkers, rejection, relayed, + Server, + Session, serveNotes, type TestPrincipal, testCodec, @@ -142,8 +144,9 @@ forEachCodec(() => { const result = await raw.next("result"); expect(result).toMatchObject({ kind: "result", call: 5 }); expect(result.origin).toBeUndefined(); - raw.send({ kind: "retry", hold: 7 }); - expect(await worker.next("retry")).toEqual({ kind: "retry", origin: 1, hold: 7 }); + raw.send({ kind: "retry", hold: 7, hops: 5 }); + // Forwarded with its budget as it is: a relay visits no object. + expect(await worker.next("retry")).toEqual({ kind: "retry", origin: 1, hold: 7, hops: 5 }); }); it("give each downstream link its own origin, never reused", async () => { @@ -350,8 +353,8 @@ forEachCodec(() => { first.close(); expect(await rejection(call)).toMatchObject({ code: "unavailable" }); await transported(); + // The link had completed its handshake: its address resolves again at once (no timer, slice 10 R2). expect(notes.state.value).toEqual({ kind: "loading" }); - await vi.advanceTimersByTimeAsync(1000); const second = await workerAt(workers, 1); const again = await second.next("hold"); expect(again).toMatchObject({ hold: held.hold, origin: held.origin }); @@ -364,7 +367,13 @@ forEachCodec(() => { }); it("answers a call that crossed the loading notice unavailable (race 13)", async () => { - const { raw, workers } = await rawRelay(); + // The address's next resolve never settles: the relayed hold has no upstream link when the call arrives. + const workers: RawPeer[] = []; + const resolve = rawWorkers(workers); + const relay = await relayed({ + resolve: (address) => (workers.length === 0 ? resolve(address) : new Promise(() => {})), + }); + const raw = rawTab(relay.server); holdPlaced(raw, 0); const worker = await workerAt(workers); bindPlaced(worker, await worker.next("hold")); @@ -379,23 +388,48 @@ forEachCodec(() => { describe("a parent link that is down (audit)", () => { it("holds a hold arriving meanwhile on the parent path: it waits loading and binds on the next parent link", async () => { - const server = createHost({ principal: { kind: "server" }, onError: () => {} }); + // The middle host relays its tab's holds to its parent, which reaches it over a route: origins arrive only on + // parent and route links (decision "Origins only from relays"). + const middle = createHost({ + address: Session({ id: "M" }), + principal: { kind: "worker", session: "M" }, + onError: () => {}, + }); + await middle.load([], context); + const server = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); + const links: Link[] = []; + let gate = Promise.resolve(); + server.route( + Session, + { + authorize: () => true, + resolve: async () => { + await gate; + const [a, b] = memoryLinkPair(); + middle.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); + links.push(a); + return { link: a, principal: { kind: "worker", session: "M" }, codec: testCodec() }; + }, + }, + context, + ); await server.load( [{ name: "@test/app", facet: defineFacet((h) => void serveNotes(h)) }], context, ); - const middle = createHost({ principal: { kind: "server" }, onError: () => {} }); - await middle.load([], context); - /** Connects the middle host to the server over a new link; returns the server's end. */ - const up = (): Link => { - const [a, b] = memoryLinkPair(); - server.accept(a, { principal: { kind: "owner" }, codec: testCodec() }, context); - middle.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); - return a; - }; - const first = up(); + // The server holds something at the middle's address: its route link exists while it does. + const anchor = server.hold(PlacedNotes, { session: "M" }, context); await transported(); - first.close(); + // The next resolve waits: the middle's parent path is down meanwhile. + let admit: () => void = () => {}; + gate = new Promise((resolve) => { + admit = resolve; + }); + links[0]!.close(); await transported(); const tab = createHost({ onError: () => {} }); await tab.load([], context); @@ -405,9 +439,11 @@ forEachCodec(() => { const notes = tab.hold(Notes, { session: "S1" }, context); await transported(); expect(notes.state.value).toEqual({ kind: "loading" }); - up(); + admit(); + await transported(); await transported(); expect(notes.state.value).toEqual({ kind: "live" }); + anchor.release(); }); }); @@ -441,9 +477,12 @@ forEachCodec(() => { bindPlaced(worker, sent); worker.send(countState([sent], 1)); await transported(); - // The relay no longer relays the hold: dropped there (Q1), each message acknowledged to the worker at once. + // The origin held nothing more: its record went and the worker was told `closed` (decision "Origins only from + // relays"); what crossed it is dropped (Q1). expect(raw.kinds()).toEqual(["hello"]); - expect(worker.received.filter(({ kind }) => kind === "ack")).toHaveLength(2); + expect(worker.received.filter(({ kind }) => kind === "closed")).toEqual([ + { kind: "closed", origin: sent.origin }, + ]); }); }); diff --git a/packages/chord/test/next/remote/remote-object.test.ts b/packages/chord/test/next/remote/remote-object.test.ts index 03baaa01c..3141dbc84 100644 --- a/packages/chord/test/next/remote/remote-object.test.ts +++ b/packages/chord/test/next/remote/remote-object.test.ts @@ -35,6 +35,7 @@ import { rawParent, rawProvider, rejection, + Server, serveNotes, type TestPrincipal, testCodec, @@ -48,7 +49,12 @@ const Counter = defineState("counter", { schema: obj({ n: int() }), version: 1, forEachCodec(() => { describe("a hold over the parent link", () => { it("reject a broken failure at once", async () => { - const Broken = remote("app.broken", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + const Broken = remote("app.broken", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const { client } = await linked((h) => h.serve(Broken, () => { throw new Error("factory threw"); @@ -197,6 +203,7 @@ forEachCodec(() => { const Flaky = remote("app.flaky", { version: 1, params: { id: int() }, + hostAddress: () => Server(), variants: { v: { state: { n: int() } } }, }); const opens: string[] = []; @@ -250,6 +257,7 @@ forEachCodec(() => { const Halves = remote("app.halves", { version: 1, params: {}, + hostAddress: () => Server(), variants: { v: { state: { half: int() }, methods: { set: method(int()) } } }, }); const { server, client } = await linked((h) => @@ -294,16 +302,20 @@ forEachCodec(() => { }); }); - describe("self.get of a type over the link (R18)", () => { - it("a failed hold fails the dependent transiently; bound retries it at once with a new hold", async () => { + describe("self.get of a type over the link (dependency model §3)", () => { + it("a failed hold parks the dependent failed { waiting }: no new attempt or hold; bound wakes it", async () => { const View = local("app.view", { version: 1, params: {}, members: { count: int() } }); const errors: HostError[] = []; const client = createHost({ onError: (error) => errors.push(error) }); const [a, b] = memoryLinkPair(); client.connect(a, { principal: { kind: "server" }, codec: testCodec() }, context); const peer = new RawPeer(b); + let factories = 0; const facet = defineFacet((h) => - h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), + h.serve(View, (_, self) => { + factories++; + return { count: self.get(Notes, { session: "S1" }).count }; + }), ); const loading = client.load([{ name: "@test/view", facet }], context); const { hold } = await peer.next("hold"); @@ -312,25 +324,20 @@ forEachCodec(() => { hold, cause: { kind: "transient", error: { kind: "chord", error: { code: "unavailable", message: "warming" } } }, }); + // Parked on a remote blocker: its first outcome, so the start barrier is over. await loading; - expect(errors).toMatchObject([ - { - kind: "activation_failed", - object: "app.view", - failure: { kind: "transient", error: { code: "unavailable" } }, - }, - ]); - const members = [ - { member: EDITOR.state.notes!, value: "[]" }, - { member: EDITOR.state.count!, value: "0" }, - ]; - peer.send({ kind: "bound", holds: [{ hold, variant: EDITOR.variant, members, failed: [] }] }); - // The retry binds its own hold before the failed attempt's is released. - const retried = await bindNext(peer, [{ text: "a" }]); - expect(retried).toBe(1); - expect(await peer.next("release")).toMatchObject({ hold }); - const view = await client.get(View, {}, context); + await transported(); + expect(errors).toEqual([]); + const view = client.hold(View, {}, context); + expect(view.state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.notes(S1): warming" } }, + }); + bindHold(peer, hold, [{ text: "a" }]); + await transported(); + expect(view.state.value).toEqual({ kind: "live" }); expect(view.count.value).toBe(1); + expect([factories, peer.kinds().filter((kind) => kind === "hold" || kind === "release")]).toEqual([1, []]); }); }); @@ -362,8 +369,13 @@ forEachCodec(() => { // Timers stand still: neither the backoff window nor a readyTimeout ends while the test runs. fakeTimers(); - it("fails at once with the real cause when its load holds it (no wait for a timeout)", async () => { - const Flaky = remote("app.flaky", { version: 1, params: {}, variants: { v: { state: { n: int() } } } }); + it("parks at once on the real cause when its load holds it (no wait for a timeout)", async () => { + const Flaky = remote("app.flaky", { + version: 1, + params: {}, + hostAddress: () => Server(), + variants: { v: { state: { n: int() } } }, + }); const View = local("app.view", { version: 1, params: {}, members: { n: int() } }); const { server, client: first } = await linked((h) => h.serve(Flaky, (_, _principal, self) => { @@ -386,16 +398,12 @@ forEachCodec(() => { second.connect(b, { principal: { kind: "server" }, codec: testCodec() }, context); const facet = defineFacet((h) => h.serve(View, (_, self) => ({ n: self.get(Flaky, {}).n }))); await second.load([{ name: "@test/view", facet }], context); - expect(errors).toMatchObject([ - { - kind: "activation_failed", - object: "app.view", - failure: { - kind: "transient", - error: { message: "app.view: dependency app.flaky failed", cause: { message: "app.flaky: offline" } }, - }, - }, - ]); + await transported(); + expect(errors).toEqual([]); + expect(second.hold(View, {}, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting", error: { message: "waiting for app.flaky: app.flaky: offline" } }, + }); await first.close(context); await failing; }); @@ -406,6 +414,7 @@ forEachCodec(() => { const Shifting = remote("app.shifting", { version: 1, params: {}, + hostAddress: () => Server(), variants: { a: { state: { x: int(), y: int() }, methods: { m: method() } }, b: { state: { y: int(), z: int() } }, @@ -461,7 +470,7 @@ forEachCodec(() => { ]); }); - it("apply bound in one batch: each member gets one frame [['r', value]] and a fresh decoder", async () => { + it("apply bound in one batch: each member gets one frame (a diff from its last value) and a fresh decoder", async () => { const { client, peer } = await rawProvider(); const notes = client.hold(Notes, { session: "S1" }, context); await bindNext(peer, [{ text: "a" }]); @@ -474,7 +483,8 @@ forEachCodec(() => { const hold = await bindNext(next, [{ text: "a" }, { text: "b" }]); await delivered(); const list = [{ text: "a" }, { text: "b" }]; - expect(frames).toEqual([[list, [["r", list]]]]); + // One frame each, a diff from the last value the reference kept while its hold re-bound. + expect(frames).toEqual([[list, [["p", [], 1, 0, [{ text: "b" }]]]]]); expect(counts).toEqual([[2, [["r", 2]]]]); // The fresh decoder: the provider's new encoder starts over, interning paths anew. next.send({ @@ -592,10 +602,15 @@ forEachCodec(() => { const NotesV2 = remote("app.notes", { version: 2, params: { session: str() }, + hostAddress: () => Server(), variants: { editor: { state: { count: int() } } }, }); const { server, client, links } = await linked((h) => void serveNotes(h)); - const upgraded = createHost({ principal: { kind: "server" }, onError: () => {} }); + const upgraded = createHost({ + address: Server(), + principal: { kind: "server" }, + onError: () => {}, + }); const facet = defineFacet((h) => h.serve(NotesV2, () => ({ variant: "editor", members: { count: constant(0) } })), ); @@ -614,7 +629,7 @@ forEachCodec(() => { const toUpgraded = await relink(upgraded, links[0]); expect(notes.state.value).toMatchObject({ kind: "failed", - cause: { kind: "broken", error: { code: "contract_mismatch" } }, + cause: { kind: "waiting", error: { code: "contract_mismatch" } }, retryAt: undefined, }); notes.retry(); @@ -633,7 +648,7 @@ forEachCodec(() => { await transported(); expect(notes.state.value).toMatchObject({ kind: "failed", cause: { kind: "broken" } }); notes.retry(); - expect(await peer.next("retry")).toEqual({ kind: "retry", hold }); + expect(await peer.next("retry")).toEqual({ kind: "retry", hold, hops: 16 }); bindHold(peer, hold); await transported(); expect(notes.state.value).toEqual({ kind: "live" }); @@ -644,6 +659,7 @@ forEachCodec(() => { const Flaky = remote("app.flaky", { version: 1, params: {}, + hostAddress: () => Server(), variants: { only: { state: {}, methods: { ping: method() } } }, }); let factories = 0; @@ -698,7 +714,7 @@ forEachCodec(() => { expect(notes.state.value).toEqual({ kind: "live" }); }); - it("a member watch before the first binding waits for it, and rejects when it fails or is released (race 16)", async () => { + it("a member watch before the first binding waits for it through a failure, and rejects when released (race 16)", async () => { const { client, peer } = await rawProvider(); const bound = client.hold(Notes, { session: "S1" }, context); const failing = client.hold(Notes, { session: "S2" }, context); @@ -715,8 +731,17 @@ forEachCodec(() => { }); released.release(); expect((await watches[0]!).value).toBe(1); - expect(await rejection(watches[1]!)).toEqual(error); expect(await rejection(watches[2]!)).toEqual({ code: "released", message: "app.notes: released" }); + // A failure is waited through (Opus G3): the watch rejects only when its reference ends. + let settled = false; + const settle = (): void => { + settled = true; + }; + void watches[1]!.then(settle, settle); + await transported(); + expect([failing.state.value.kind, settled]).toEqual(["failed", false]); + failing.release(); + expect(await rejection(watches[1]!)).toMatchObject({ code: "released" }); }); }); @@ -751,15 +776,6 @@ forEachCodec(() => { h.serve(View, (_, self) => ({ count: self.get(Notes, { session: "S1" }).count })), ); - /** Binds the kept hold (0) on `next`; the retry's own hold (1) is bound and the kept one released (R18). */ - async function retried(client: Host, next: RawPeer): Promise { - expect(await bindNext(next, [{ text: "a" }])).toBe(0); - // Retried at once, long before its backoff window (1 s) ends: its new attempt holds anew. - expect(await bindNext(next, [{ text: "a" }])).toBe(1); - expect(await next.next("release")).toEqual({ kind: "release", hold: 0 }); - expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); - } - it("stays loading through the drop, no failure reported, and goes live when the next link binds it", async () => { const errors: HostError[] = []; const client = createHost({ onError: (error) => errors.push(error) }); @@ -778,28 +794,34 @@ forEachCodec(() => { expect(errors).toEqual([]); }); - it("dropped and never bound again, fails transient at its readyTimeout (the bound of the wait)", async () => { + it("dropped and never bound again, stays parked loading: nothing fails, one parked_long (audit A11)", async () => { const errors: HostError[] = []; const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); const peer = await rawParent(client); const loading = client.load([{ name: "@test/view", facet }], context); await peer.next("hold"); + // A remote blocker ends the start barrier's first pass. + await loading; peer.close(); await transported(); - expect(errors).toEqual([]); - await vi.advanceTimersByTimeAsync(1_000); - await loading; - expect(failures(errors)).toEqual(["app.view: transient"]); + await vi.advanceTimersByTimeAsync(10_000); + // A diagnostic whatever the blocker (audit A11): its path without a link too; nothing failed. + expect(errors).toEqual([ + { kind: "parked_long", object: "app.view", waitingFor: "app.notes(S1)", state: "loading", since: 0 }, + ]); + expect(client.hold(View, {}, context).state.value).toEqual({ kind: "loading" }); }); - it("without any link fails at its readyTimeout, keeps the reference, and is retried when the first link binds it", async () => { + it("without any link stays loading past its readyTimeout and goes live when the first link binds its one hold", async () => { const errors: HostError[] = []; const client = createHost({ readyTimeout: 1_000, onError: (error) => errors.push(error) }); - const loading = client.load([{ name: "@test/view", facet }], context); - await vi.advanceTimersByTimeAsync(1_000); - await loading; - expect(failures(errors)).toEqual(["app.view: transient"]); - await retried(client, await rawParent(client)); + await client.load([{ name: "@test/view", facet }], context); + await vi.advanceTimersByTimeAsync(500); + expect(failures(errors)).toEqual([]); + const next = await rawParent(client); + expect(await bindNext(next, [{ text: "a" }])).toBe(0); + expect((await client.get(View, {}, context, { readyTimeout: 1 })).count.value).toBe(1); + expect(next.kinds().filter((kind) => kind === "hold" || kind === "release")).toEqual([]); }); }); diff --git a/packages/chord/test/next/schema/declarations.ts b/packages/chord/test/next/schema/declarations.ts new file mode 100644 index 000000000..cf0a329c0 --- /dev/null +++ b/packages/chord/test/next/schema/declarations.ts @@ -0,0 +1,180 @@ +// The declarations of the schema switch's proofs (TypeBox to pi-schema): one of every schema helper and option in a +// contract, persisted documents, tables with defaults, an optional and a `json(...)` column, an evolved table and a +// persisted job. Their fingerprints were recorded with the TypeBox helpers; the stored data in `stored/` was written +// by that code (`write-stored.ts`). + +import { + arr, + bool, + bytes, + hostKind, + int, + keyed, + lit, + local, + map, + method, + nul, + num, + obj, + oneOf, + opt, + remote, + str, + union, + unknown, +} from "../../../src/next/contracts/index.ts"; +import { defineState } from "../../../src/next/documents/index.ts"; +import { defineJob } from "../../../src/next/jobs/index.ts"; +import { defineTable, evolveTable, json } from "../../../src/next/store/index.ts"; + +/** A schema with every helper and option a declaration can use. */ +export const everything = obj({ + name: str({ min: 1, max: 40, pattern: "^[a-z ]+$" }), + nick: str({ default: "x" }), + age: int({ min: 0, max: 150 }), + count: int({ default: 0 }), + score: num({ min: 0, max: 100 }), + ratio: num({ default: 0.5 }), + active: bool(), + verified: bool({ default: false }), + role: oneOf("admin", "user", "guest"), + level: oneOf(1, 2, 3), + kind: lit("person"), + one: lit(1), + yes: lit(true), + tags: arr(str(), { min: 0, max: 5 }), + points: arr(int()), + extra: map(int()), + note: opt(str()), + nothing: nul(), + choice: union([str(), int(), obj({ deep: arr(obj({ x: num() })) })]), + nested: obj({ inner: opt(obj({ v: str() })) }), +}); + +/** A local contract with state, keyed state and methods of every argument shape. */ +export const Everything = local("fixture.switch.everything", { + version: 1, + params: { id: str(), shard: int() }, + members: { + profile: everything, + notes: keyed(obj({ body: str() })), + plain: method(), + args: method(obj({ a: str(), b: opt(int()), any: unknown() })), + result: method(undefined, arr(str())), + both: method(obj({ when: int({ min: 0 }) }), union([lit("ok"), obj({ error: str() })])), + blob: method(obj({ data: bytes({ max: 1024 }) }), bytes()), + }, +}); + +const Server = hostKind("fixture.switch.server", { region: str() }); + +/** A remote contract with variants and a host address. */ +export const Account = remote("fixture.switch.account", { + version: 1, + params: { user: str({ min: 1 }) }, + hostAddress: () => Server({ region: "eu" }), + variants: { + owner: { state: { balance: num() }, methods: { pay: method(obj({ amount: num({ min: 0 }) }), bool()) } }, + guest: { state: { balance: num() } }, + }, +}); + +/** A persisted singleton document of `everything`. */ +export const Profile = defineState("fixture.switch.profile", { + schema: everything, + version: 1, + initial: () => ({ + name: "ada", + nick: "x", + age: 36, + count: 0, + score: 1, + ratio: 0.5, + active: true, + verified: false, + role: "admin" as const, + level: 1 as const, + kind: "person" as const, + one: 1 as const, + yes: true as const, + tags: [], + points: [], + extra: {}, + nothing: null, + choice: "a", + nested: {}, + }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 2, +}); + +/** A persisted keyed document. */ +export const Notes = defineState("fixture.switch.notes", { + schema: obj({ body: str(), tags: opt(arr(str())) }), + version: 1, + initial: () => ({ body: "" }), + keyed: true, + persist: true, + checkpointWhen: () => false, +}); + +/** A table with defaults, an optional column and a `json(...)` column. */ +export const Rows = defineTable("fixture.switch.rows", { + version: 1, + columns: { + id: str(), + n: int({ default: 0 }), + flag: bool({ default: false }), + label: opt(str()), + data: json(obj({ a: int(), b: opt(arr(str())) }), { version: 1 }), + maybe: opt(json(map(num()), { version: 1 })), + }, + key: ["id"], + indexes: { by_n: ["n", "id"] }, +}); + +/** A table evolved over three versions: a rename, an added column with a default, a json column, a drop. */ +export const EditsV1 = defineTable("fixture.switch.edits", { + version: 1, + columns: { id: str(), file: str(), at: int() }, + key: ["id"], +}); + +export const EditsV2 = evolveTable(EditsV1, (t) => + t + .rename("file", "path") + .add("dir", str({ default: "" })) + .rows((row) => ({ dir: row.path.slice(0, row.path.lastIndexOf("/")) })), +); + +export const EditsV3 = evolveTable(EditsV2, (t) => t.drop("dir").index("by_path", ["path", "at"])); + +/** A persisted job with a state document. */ +export const ImportState = defineState("fixture.switch.import.state", { + schema: obj({ done: int() }), + version: 1, + initial: () => ({ done: 0 }), + persist: true, + checkpointWhen: () => false, +}); + +export const ImportJob = defineJob("fixture.switch.import", { + version: 1, + args: obj({ path: str() }), + result: obj({ count: int() }), + state: ImportState, + persist: { resume: "rerun" }, +}); + +/** The object owning the persisted documents and the tables (a local type with empty params, as tables need). */ +export const Owner = local("fixture.switch.owner", { + version: 1, + params: {}, + members: { + profile: everything, + write: method(obj({ id: str(), body: str() })), + read: method(str(), str()), + dump: method(obj({}), unknown()), + }, +}); diff --git a/packages/chord/test/next/schema/owner.ts b/packages/chord/test/next/schema/owner.ts new file mode 100644 index 000000000..88928df5e --- /dev/null +++ b/packages/chord/test/next/schema/owner.ts @@ -0,0 +1,85 @@ +// The object of the stored-data proof: it owns the persisted documents, the tables and the job of `declarations.ts`, +// writes them (`write-stored.ts`, with the TypeBox helpers) and reads them back (the test, after the switch). + +import type { Context } from "../../../src/context/index.ts"; +import { createHost, defineFacet, type Host } from "../../../src/next/host/index.ts"; +import type { TableDefinition } from "../../../src/next/store/index.ts"; +import { openStore, query, type Store, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import { ImportJob, Notes, Owner, Profile, Rows } from "./declarations.ts"; + +/** What the job's `run` does in this process: wait until its context ends, or finish at once. */ +export type RunBehaviour = "wait" | "finish"; + +/** The finished runs' results, in order. */ +export const finished: unknown[] = []; + +/** The owner's facet over `edits` (the table version this process declares). */ +export function ownerFacet(edits: TableDefinition, behaviour: RunBehaviour) { + return defineFacet((host) => { + host.serve(Owner, (_, self) => { + const profile = self.state(Profile); + const notes = self.state(Notes); + const rows = self.table(Rows); + const editsTable = self.table(edits); + const imports = self.job(ImportJob, { + run: async ({ path }, ctx) => { + await ctx.state.change((draft) => { + draft.done += path.length; + }, ctx); + if (behaviour === "wait") { + await new Promise((resolve) => ctx.abortSignal?.addEventListener("abort", resolve)); + } + return { count: ctx.state.value.done }; + }, + finish: async (outcome) => { + finished.push(outcome.kind === "done" ? outcome.result : outcome.kind); + }, + }); + return { + profile, + write: async ({ id, body }, ctx) => { + await self.tx(async (tx) => { + (await tx.draft(notes, id)).body = body; + const draft = await tx.draft(profile); + draft.count += 1; + draft.tags.push(id); + draft.extra[id] = body.length; + draft.choice = { deep: [{ x: draft.count }] }; + draft.nested = { inner: { v: body } }; + tx.put(rows, { + id, + label: body, + data: { a: body.length, b: [id] }, + ...(id === "a" ? { maybe: { n: 1.5 } } : {}), + }); + tx.put(editsTable, { id, file: `/notes/${id}.md`, at: body.length } as never); + }, ctx); + await imports.start({ path: id }, ctx); + return null; + }, + read: async (id, ctx) => (await notes.get(id, ctx))?.body ?? "", + dump: async (_, ctx) => ({ + profile: profile.value, + notes: { a: (await notes.get("a", ctx)) ?? null, b: (await notes.get("b", ctx)) ?? null }, + rows: (await rows.page(query(Rows), 10, ctx)).items, + edits: (await editsTable.page(query(edits as TableDefinition), 10, ctx)).items, + runs: imports.running.value.length, + }), + }; + }); + }); +} + +/** A host serving the owner on the SQLite store at `path`. */ +export async function ownerHost( + path: string, + edits: TableDefinition, + behaviour: RunBehaviour, + context: Context, +): Promise<{ store: Store; host: Host }> { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context); + const host = createHost({ store, onError: () => {}, closeTimeout: 100 }); + await host.load([{ name: "@fixture/owner", facet: ownerFacet(edits, behaviour) }], context); + return { store, host }; +} diff --git a/packages/chord/test/next/schema/proofs.ts b/packages/chord/test/next/schema/proofs.ts new file mode 100644 index 000000000..6f285111e --- /dev/null +++ b/packages/chord/test/next/schema/proofs.ts @@ -0,0 +1,113 @@ +// The values the schema switch must keep (`recorded.json`, made with the TypeBox helpers): the fingerprints and normal +// forms of `declarations.ts`, and what a host reads back from the stored data in `stored/`. + +import { createReadStream } from "node:fs"; +import { copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Context } from "../../../src/context/index.ts"; +import type { JsonValue } from "../../../src/json.ts"; +import type { Contract } from "../../../src/next/contracts/index.ts"; +import { contractData } from "../../../src/next/contracts/internal.ts"; +import { snapshotSchema } from "../../../src/next/contracts/snapshot.ts"; +import { decodeJsonl, importStore, sqliteBackend } from "../../../src/next/store/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; +import * as declared from "./declarations.ts"; +import { finished, ownerHost } from "./owner.ts"; + +/** Every fingerprint and normal form of the declarations. */ +export function fingerprints(): JsonValue { + const contract = (C: Contract) => { + const data = contractData(C); + return { + params: data.params.normal, + versions: data.versions.map((version) => ({ + fingerprint: version.fingerprint, + variants: Object.fromEntries( + [...version.variants].map(([name, variant]) => [ + name, + { + state: Object.fromEntries([...variant.state].map(([m, s]) => [m, s.normal])), + keyed: Object.fromEntries([...variant.keyed].map(([m, s]) => [m, s.normal])), + methods: Object.fromEntries( + [...variant.methods].map(([m, { args, result }]) => [ + m, + { args: args.normal, result: result.normal }, + ]), + ), + }, + ]), + ), + })), + }; + }; + const table = (T: { readonly fingerprint: string; readonly record: unknown }) => ({ + fingerprint: T.fingerprint, + record: T.record as unknown as JsonValue, + }); + return { + contracts: { + everything: contract(declared.Everything), + account: contract(declared.Account), + owner: contract(declared.Owner), + }, + documents: { + profile: { + fingerprint: declared.Profile.fingerprint, + normal: snapshotSchema(declared.everything, "p").normal, + }, + notes: declared.Notes.fingerprint, + importState: declared.ImportState.fingerprint, + }, + tables: { + rows: table(declared.Rows), + editsV1: table(declared.EditsV1), + editsV2: table(declared.EditsV2), + editsV3: table(declared.EditsV3), + }, + job: declared.ImportJob.fingerprint, + }; +} + +const stored = fileURLToPath(new URL("./stored/", import.meta.url)); + +/** + * What a host of the current code reads from the stored data: from a copy of the SQLite store, and from a store + * imported from the archive (both written at table version 1, read at version 3; the stopped job runs resume). + */ +export async function storedReads(context: Context): Promise { + const directory = await mkdtemp(join(tmpdir(), "chord-schema-switch-")); + try { + const fromStore = join(directory, "store.db"); + await copyFile(join(stored, "store.db"), fromStore); + const store = await read(fromStore, context); + const imported = await importStore( + sqliteBackend(await openNodeSqliteDatabase(join(directory, "imported.db"))), + decodeJsonl(createReadStream(join(stored, "archive.jsonl"))), + context, + ); + await imported.close(context); + const archive = await read(join(directory, "imported.db"), context); + return { store, archive }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** The owner's dump on the store at `path`, after its stopped runs finished. */ +async function read(path: string, context: Context): Promise { + finished.length = 0; + const { store, host } = await ownerHost(path, declared.EditsV3, "finish", context); + const owner = await host.get(declared.Owner, {}, context); + for (let i = 0; i < 100 && finished.length < 3; i++) await new Promise((resolve) => setTimeout(resolve, 10)); + const dump = await owner.dump({}, context); + const result = { + dump: dump as JsonValue, + finished: [...finished] as JsonValue, + note: await owner.read("b", context), + }; + await host.close(context); + await store.close(context); + return result; +} diff --git a/packages/chord/test/next/schema/recorded.json b/packages/chord/test/next/schema/recorded.json new file mode 100644 index 000000000..00c0a3588 --- /dev/null +++ b/packages/chord/test/next/schema/recorded.json @@ -0,0 +1,1143 @@ +{ + "fingerprints": { + "contracts": { + "everything": { + "params": [ + [ + "id", + { + "type": "string" + } + ], + [ + "shard", + { + "type": "integer" + } + ] + ], + "versions": [ + { + "fingerprint": "3861c5a457ef4aa3", + "variants": { + "members": { + "state": { + "profile": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "keyed": { + "notes": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + } + }, + "required": [ + "body" + ], + "type": "object" + } + }, + "methods": { + "plain": { + "args": { + "type": "null" + }, + "result": { + "type": "null" + } + }, + "args": { + "args": { + "additionalProperties": false, + "properties": { + "a": { + "type": "string" + }, + "any": {}, + "b": { + "type": "integer" + } + }, + "required": [ + "a", + "any" + ], + "type": "object" + }, + "result": { + "type": "null" + } + }, + "result": { + "args": { + "type": "null" + }, + "result": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "both": { + "args": { + "additionalProperties": false, + "properties": { + "when": { + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "when" + ], + "type": "object" + }, + "result": { + "anyOf": [ + { + "const": "ok", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "type": "object" + } + ] + } + }, + "blob": { + "args": { + "additionalProperties": false, + "properties": { + "data": { + "maxLength": 1024, + "type": "bytes" + } + }, + "required": [ + "data" + ], + "type": "object" + }, + "result": { + "type": "bytes" + } + } + } + } + } + } + ] + }, + "account": { + "params": [ + [ + "user", + { + "minLength": 1, + "type": "string" + } + ] + ], + "versions": [ + { + "fingerprint": "821be6a79fbe350a", + "variants": { + "owner": { + "state": { + "balance": { + "type": "number" + } + }, + "keyed": {}, + "methods": { + "pay": { + "args": { + "additionalProperties": false, + "properties": { + "amount": { + "minimum": 0, + "type": "number" + } + }, + "required": [ + "amount" + ], + "type": "object" + }, + "result": { + "type": "boolean" + } + } + } + }, + "guest": { + "state": { + "balance": { + "type": "number" + } + }, + "keyed": {}, + "methods": {} + } + } + } + ] + }, + "owner": { + "params": [], + "versions": [ + { + "fingerprint": "b01eec2d4e2311b0", + "variants": { + "members": { + "state": { + "profile": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "keyed": {}, + "methods": { + "write": { + "args": { + "additionalProperties": false, + "properties": { + "body": { + "type": "string" + }, + "id": { + "type": "string" + } + }, + "required": [ + "body", + "id" + ], + "type": "object" + }, + "result": { + "type": "null" + } + }, + "read": { + "args": { + "type": "string" + }, + "result": { + "type": "string" + } + }, + "dump": { + "args": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "result": {} + } + } + } + } + } + ] + } + }, + "documents": { + "profile": { + "fingerprint": "f345152aff1bb993", + "normal": { + "additionalProperties": false, + "properties": { + "active": { + "type": "boolean" + }, + "age": { + "maximum": 150, + "minimum": 0, + "type": "integer" + }, + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + }, + { + "additionalProperties": false, + "properties": { + "deep": { + "items": { + "additionalProperties": false, + "properties": { + "x": { + "type": "number" + } + }, + "required": [ + "x" + ], + "type": "object" + }, + "type": "array" + } + }, + "required": [ + "deep" + ], + "type": "object" + } + ] + }, + "count": { + "default": 0, + "type": "integer" + }, + "extra": { + "patternProperties": { + "^.*$": { + "type": "integer" + } + }, + "type": "object" + }, + "kind": { + "const": "person", + "type": "string" + }, + "level": { + "anyOf": [ + { + "const": 1, + "type": "number" + }, + { + "const": 2, + "type": "number" + }, + { + "const": 3, + "type": "number" + } + ] + }, + "name": { + "maxLength": 40, + "minLength": 1, + "pattern": "^[a-z ]+$", + "type": "string" + }, + "nested": { + "additionalProperties": false, + "properties": { + "inner": { + "additionalProperties": false, + "properties": { + "v": { + "type": "string" + } + }, + "required": [ + "v" + ], + "type": "object" + } + }, + "type": "object" + }, + "nick": { + "default": "x", + "type": "string" + }, + "note": { + "type": "string" + }, + "nothing": { + "type": "null" + }, + "one": { + "const": 1, + "type": "number" + }, + "points": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "ratio": { + "default": 0.5, + "type": "number" + }, + "role": { + "anyOf": [ + { + "const": "admin", + "type": "string" + }, + { + "const": "user", + "type": "string" + }, + { + "const": "guest", + "type": "string" + } + ] + }, + "score": { + "maximum": 100, + "minimum": 0, + "type": "number" + }, + "tags": { + "items": { + "type": "string" + }, + "maxItems": 5, + "minItems": 0, + "type": "array" + }, + "verified": { + "default": false, + "type": "boolean" + }, + "yes": { + "const": true, + "type": "boolean" + } + }, + "required": [ + "active", + "age", + "choice", + "count", + "extra", + "kind", + "level", + "name", + "nested", + "nick", + "nothing", + "one", + "points", + "ratio", + "role", + "score", + "tags", + "verified", + "yes" + ], + "type": "object" + } + }, + "notes": "bd44e8ea54b8f283", + "importState": "53d035820123516d" + }, + "tables": { + "rows": { + "fingerprint": "a8a7a46b1d1b5058", + "record": { + "id": { + "type": "str" + }, + "n": { + "type": "int", + "default": 0 + }, + "flag": { + "type": "bool", + "default": false + }, + "label": { + "type": "str", + "optional": true + }, + "data": { + "type": "json", + "version": 1, + "fingerprint": "595acb546ede10d5" + }, + "maybe": { + "type": "json", + "optional": true, + "version": 1, + "fingerprint": "27fc51c99dad9a88" + } + } + }, + "editsV1": { + "fingerprint": "b6a64f82180e9b7f", + "record": { + "id": { + "type": "str" + }, + "file": { + "type": "str" + }, + "at": { + "type": "int" + } + } + }, + "editsV2": { + "fingerprint": "2fbd0130077d5eed", + "record": { + "id": { + "type": "str" + }, + "path": { + "type": "str" + }, + "at": { + "type": "int" + }, + "dir": { + "type": "str", + "default": "" + } + } + }, + "editsV3": { + "fingerprint": "303055ac505946c3", + "record": { + "id": { + "type": "str" + }, + "path": { + "type": "str" + }, + "at": { + "type": "int" + } + } + } + }, + "job": "c280624818bc83ab" + }, + "stored": { + "store": { + "dump": { + "profile": { + "name": "ada", + "nick": "x", + "age": 36, + "count": 3, + "score": 1, + "ratio": 0.5, + "active": true, + "verified": false, + "role": "admin", + "level": 1, + "kind": "person", + "one": 1, + "yes": true, + "tags": [ + "a", + "b", + "c" + ], + "points": [], + "extra": { + "a": 10, + "b": 6, + "c": 9 + }, + "nothing": null, + "choice": { + "deep": [ + { + "x": 3 + } + ] + }, + "nested": { + "inner": { + "v": "third one" + } + } + }, + "notes": { + "a": { + "body": "first note" + }, + "b": { + "body": "second" + } + }, + "rows": [ + { + "id": "a", + "n": 0, + "flag": false, + "label": "first note", + "data": { + "a": 10, + "b": [ + "a" + ] + }, + "maybe": { + "n": 1.5 + } + }, + { + "id": "b", + "n": 0, + "flag": false, + "label": "second", + "data": { + "a": 6, + "b": [ + "b" + ] + } + }, + { + "id": "c", + "n": 0, + "flag": false, + "label": "third one", + "data": { + "a": 9, + "b": [ + "c" + ] + } + } + ], + "edits": [ + { + "id": "a", + "path": "/notes/a.md", + "at": 10 + }, + { + "id": "b", + "path": "/notes/b.md", + "at": 6 + }, + { + "id": "c", + "path": "/notes/c.md", + "at": 9 + } + ], + "runs": 0 + }, + "finished": [ + { + "count": 2 + }, + { + "count": 2 + }, + { + "count": 2 + } + ], + "note": "second" + }, + "archive": { + "dump": { + "profile": { + "name": "ada", + "nick": "x", + "age": 36, + "count": 3, + "score": 1, + "ratio": 0.5, + "active": true, + "verified": false, + "role": "admin", + "level": 1, + "kind": "person", + "one": 1, + "yes": true, + "tags": [ + "a", + "b", + "c" + ], + "points": [], + "extra": { + "a": 10, + "b": 6, + "c": 9 + }, + "nothing": null, + "choice": { + "deep": [ + { + "x": 3 + } + ] + }, + "nested": { + "inner": { + "v": "third one" + } + } + }, + "notes": { + "a": { + "body": "first note" + }, + "b": { + "body": "second" + } + }, + "rows": [ + { + "id": "a", + "n": 0, + "flag": false, + "label": "first note", + "data": { + "a": 10, + "b": [ + "a" + ] + }, + "maybe": { + "n": 1.5 + } + }, + { + "id": "b", + "n": 0, + "flag": false, + "label": "second", + "data": { + "a": 6, + "b": [ + "b" + ] + } + }, + { + "id": "c", + "n": 0, + "flag": false, + "label": "third one", + "data": { + "a": 9, + "b": [ + "c" + ] + } + } + ], + "edits": [ + { + "id": "a", + "path": "/notes/a.md", + "at": 10 + }, + { + "id": "b", + "path": "/notes/b.md", + "at": 6 + }, + { + "id": "c", + "path": "/notes/c.md", + "at": 9 + } + ], + "runs": 0 + }, + "finished": [ + { + "count": 2 + }, + { + "count": 2 + }, + { + "count": 2 + } + ], + "note": "second" + } + } +} diff --git a/packages/chord/test/next/schema/schema-switch.test.ts b/packages/chord/test/next/schema/schema-switch.test.ts new file mode 100644 index 000000000..830593422 --- /dev/null +++ b/packages/chord/test/next/schema/schema-switch.test.ts @@ -0,0 +1,22 @@ +// Tests of the schema switch from TypeBox to pi-schema (decision "A schema package beneath pi-ai and Chord"): the same +// declarations give byte-identical fingerprints and normal forms, and data stored by the code before the switch loads +// after it (`recorded.json` and `stored/` were made with the TypeBox helpers, see `write-stored.ts`). + +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { fingerprints, storedReads } from "./proofs.ts"; + +const recorded = JSON.parse(readFileSync(new URL("./recorded.json", import.meta.url), "utf8")); + +describe("fingerprints", () => { + it("keeps every fingerprint and normal form of contracts, documents, tables, json columns and jobs", () => { + expect(fingerprints()).toEqual(recorded.fingerprints); + }); +}); + +describe("stored data", () => { + it("reads a store and an archive written before the switch: documents, tables, a migrated table, stopped runs", async () => { + expect(await storedReads(context)).toEqual(recorded.stored); + }); +}); diff --git a/packages/chord/test/next/schema/stored/archive.jsonl b/packages/chord/test/next/schema/stored/archive.jsonl new file mode 100644 index 000000000..7fcbd5972 --- /dev/null +++ b/packages/chord/test/next/schema/stored/archive.jsonl @@ -0,0 +1,36 @@ +{"kind":"manifest","format":1,"schema":2,"lastSeq":13,"nextId":12} +{"kind":"table","table":"fixture.switch.owner:fixture.switch.edits","version":1,"state":"ready","columns":{"id":{"type":"str"},"file":{"type":"str"},"at":{"type":"int"}},"key":["id"],"fingerprint":"b6a64f82180e9b7f"} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"a","file":"/notes/a.md","at":10},"version":1,"columns":{}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"b","file":"/notes/b.md","at":6},"version":1,"columns":{}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.edits","row":{"id":"c","file":"/notes/c.md","at":9},"version":1,"columns":{}} +{"kind":"table","table":"fixture.switch.owner:fixture.switch.rows","version":1,"state":"ready","columns":{"id":{"type":"str"},"n":{"type":"int","default":0},"flag":{"type":"bool","default":false},"label":{"type":"str","optional":true},"data":{"type":"json","version":1,"fingerprint":"595acb546ede10d5"},"maybe":{"type":"json","optional":true,"version":1,"fingerprint":"27fc51c99dad9a88"}},"key":["id"],"fingerprint":"a8a7a46b1d1b5058"} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"a","n":0,"flag":false,"label":"first note","data":{"a":10,"b":["a"]},"maybe":{"n":1.5}},"version":1,"columns":{"data":1,"maybe":1}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"b","n":0,"flag":false,"label":"second","data":{"a":6,"b":["b"]}},"version":1,"columns":{"data":1}} +{"kind":"put","table":"fixture.switch.owner:fixture.switch.rows","row":{"id":"c","n":0,"flag":false,"label":"third one","data":{"a":9,"b":["c"]}},"version":1,"columns":{"data":1}} +{"kind":"index","table":"fixture.switch.owner:fixture.switch.rows","name":"by_n","columns":["n","id"]} +{"kind":"document.create","record":{"id":2,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"a","history":"none","createdAt":2,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":2,"seq":2,"content":{"kind":"base","version":1,"value":{"body":"first note"}}} +{"kind":"document.create","record":{"id":3,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.profile","history":"none","createdAt":2,"revision":3,"version":1,"fingerprint":"f345152aff1bb993"}} +{"kind":"document.revision","id":3,"seq":2,"content":{"kind":"base","version":1,"value":{"name":"ada","nick":"x","age":36,"count":1,"score":1,"ratio":0.5,"active":true,"verified":false,"role":"admin","level":1,"kind":"person","one":1,"yes":true,"tags":["a"],"points":[],"extra":{"a":10},"nothing":null,"choice":{"deep":[{"x":1}]},"nested":{"inner":{"v":"first note"}}}}} +{"kind":"document.revision","id":3,"seq":5,"content":{"kind":"delta","version":1,"ops":[["s",["count"],2],["s",["choice"],{"deep":[{"x":2}]}],["s",["nested"],{"inner":{"v":"second"}}],["p",["tags"],1,0,["b"]],["s",["extra","b"],6]]}} +{"kind":"document.revision","id":3,"seq":9,"content":{"kind":"delta","version":1,"ops":[["s",["count"],3],["s",["choice"],{"deep":[{"x":3}]}],["s",["nested"],{"inner":{"v":"third one"}}],["p",["tags"],2,0,["c"]],["s",["extra","c"],9]]}} +{"kind":"document.create","record":{"id":4,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176c-71f3-a32c-b9cbc67247a3","history":"none","createdAt":3,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":4,"seq":4,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"a"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":5,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176c-71f3-a32c-b9cbc67247a3","history":"none","createdAt":3,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":5,"seq":3,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":5,"seq":6,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"document.create","record":{"id":6,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"b","history":"none","createdAt":5,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":6,"seq":5,"content":{"kind":"base","version":1,"value":{"body":"second"}}} +{"kind":"document.create","record":{"id":7,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176e-71f3-a32c-b9cc7f5426c6","history":"none","createdAt":7,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":7,"seq":8,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"b"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":8,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176e-71f3-a32c-b9cc7f5426c6","history":"none","createdAt":7,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":8,"seq":7,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":8,"seq":10,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"document.create","record":{"id":9,"owner":"fixture.switch.owner","namespace":"chord","kind":"fixture.switch.notes","key":"c","history":"none","createdAt":9,"revision":1,"version":1,"fingerprint":"bd44e8ea54b8f283"}} +{"kind":"document.revision","id":9,"seq":9,"content":{"kind":"base","version":1,"value":{"body":"third one"}}} +{"kind":"document.create","record":{"id":10,"owner":"fixture.switch.owner","namespace":"chord.jobs","kind":"fixture.switch.import","key":"01a1943d-176f-71f3-a32c-b9cf6bb98dc7","history":"none","createdAt":11,"revision":2,"version":1,"fingerprint":"chord.job-run.1"}} +{"kind":"document.revision","id":10,"seq":12,"content":{"kind":"base","version":1,"value":{"version":1,"fingerprint":"c280624818bc83ab","args":{"path":"c"},"phase":"running","restarts":0,"finishFailures":0}}} +{"kind":"document.create","record":{"id":11,"owner":"fixture.switch.owner","namespace":"chord.job-states","kind":"fixture.switch.import","key":"01a1943d-176f-71f3-a32c-b9cf6bb98dc7","history":"none","createdAt":11,"revision":2,"version":1,"fingerprint":"53d035820123516d"}} +{"kind":"document.revision","id":11,"seq":11,"content":{"kind":"base","version":1,"value":{"done":0}}} +{"kind":"document.revision","id":11,"seq":13,"content":{"kind":"delta","version":1,"ops":[["s",["done"],1]]}} +{"kind":"end","writes":34} diff --git a/packages/chord/test/next/schema/stored/store.db b/packages/chord/test/next/schema/stored/store.db new file mode 100644 index 000000000..f9b1e7083 Binary files /dev/null and b/packages/chord/test/next/schema/stored/store.db differ diff --git a/packages/chord/test/next/schema/write-stored.ts b/packages/chord/test/next/schema/write-stored.ts new file mode 100644 index 000000000..e40248d32 --- /dev/null +++ b/packages/chord/test/next/schema/write-stored.ts @@ -0,0 +1,40 @@ +// Writes the stored data of the schema switch's proof (`stored/`): a SQLite store and its export archive, made by the +// code before the switch (TypeBox helpers). Kept as it was written; rerun only to make a new fixture on purpose. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/schema/write-stored.ts + +import { createWriteStream } from "node:fs"; +import { copyFile, mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pipeline } from "node:stream/promises"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { encodeJsonl } from "../../../src/next/store/index.ts"; +import { EditsV1, Owner } from "./declarations.ts"; +import { ownerHost } from "./owner.ts"; + +const stored = fileURLToPath(new URL("./stored/", import.meta.url)); +const directory = await mkdtemp(join(tmpdir(), "chord-schema-switch-")); +const path = join(directory, "store.db"); +try { + const { store, host } = await ownerHost(path, EditsV1, "wait", context); + const owner = await host.get(Owner, {}, context); + for (const [id, body] of [ + ["a", "first note"], + ["b", "second"], + ["c", "third one"], + ] as const) { + await owner.write({ id, body }, context); + } + await new Promise((resolve) => setTimeout(resolve, 200)); + await mkdir(stored, { recursive: true }); + await pipeline(encodeJsonl(store.export(context)), createWriteStream(join(stored, "archive.jsonl"))); + // A clean close: the waiting runs are stored `stopped`, resumed by the next host. + await host.close(context); + await store.close(context); + await copyFile(path, join(stored, "store.db")); + console.log("written to", stored); +} finally { + await rm(directory, { recursive: true, force: true }); +} diff --git a/packages/chord/test/next/state/native-source.test.ts b/packages/chord/test/next/state/native-source.test.ts new file mode 100644 index 000000000..cb31e8770 --- /dev/null +++ b/packages/chord/test/next/state/native-source.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { int, local, method, str } from "../../../src/next/contracts/index.ts"; +import { ChordError } from "../../../src/next/data/index.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import type { NativeWatchCall, Watchable } from "../../../src/watch.ts"; +import { delivered, failLive, flushed, instancesOf, loaded, rejection } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Count = defineRegistry("app.count", { version: 1 }); + +const Setter = local("app.setter", { version: 1, params: {}, members: { set: method(int()) } }); + +/** Serves `Count` (a sum that throws when negative) and `Setter`, which contributes a writable number. */ +function serveCount(h: Parameters[0]>[0]): void { + h.serve(Count, { + compose: (table) => { + const sum = table.reduce((n, { value }) => n + value, 0); + if (sum < 0) throw new Error(`negative: ${sum}`); + return sum; + }, + }); + h.serve(Setter, (_, self) => { + const value = self.writable(1); + self.contribute(Count, value); + return { + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); +} + +describe("native sources as derive sources", () => { + it("a derive reads JSON and native sources alike, marked when a native one changes", async () => { + const Mixed = local("app.mixed", { version: 1, params: {}, members: { label: str(), name: method(str()) } }); + const { host } = await loaded((h) => { + serveCount(h); + h.serve(Mixed, (_, self) => { + const name = self.writable("a"); + return { + label: self.derive([name, self.registry(Count)], ([n, count]) => `${n}:${count}`), + name: async (text: string) => { + name.set(text); + return null; + }, + }; + }); + }); + const mixed = await host.get(Mixed, {}, context); + await (await host.get(Setter, {}, context)).set(2, context); + expect(mixed.label.value).toBe("a:2"); + await mixed.name("b", context); + expect(mixed.label.value).toBe("b:2"); + await host.close(context); + }); + + it("`ref.state` is a derive source: a state member can show a dependency's reachability", async () => { + const Flaky = local("app.flaky", { version: 1, params: { id: int() }, members: {} }); + const Watcher = local("app.watcher", { version: 1, params: {}, members: { flaky: str() } }); + let flakySelf: Self | undefined; + const { host } = await loaded((h) => { + h.serve(Flaky, (_, self) => { + flakySelf = self; + return {}; + }); + h.serve(Watcher, (_, self) => { + const flaky = self.get(Flaky, { id: 1 }); + return { flaky: self.derive([flaky.state], ([state]) => state.kind) }; + }); + }); + const watcher = await host.get(Watcher, {}, context); + const kinds: string[] = []; + (await watcher.flaky.watch(context)).start( + async (call) => void (call.kind === "value" && kinds.push(call.value)), + ); + expect(watcher.flaky.value).toBe("live"); + failLive(objects, flakySelf!, new ChordError({ code: "unavailable", message: "down" })); + expect(watcher.flaky.value).toBe("failed"); + await delivered(); + expect(kinds).toEqual(["failed"]); + await host.close(context); + }); + + it("types a derive's sources: watchables of either kind, never a native's handle (Fable)", () => { + const declare = (self: Self): void => { + const handle = self.open( + async () => 1, + () => {}, + ); + // @ts-expect-error a native handle has a `value` but is no watchable + self.derive([handle], ([n]) => n); + expectTypeOf(self.derive([self.writable(1), constant("a")], ([n, s]) => s.length + n)).toEqualTypeOf< + Watchable + >(); + }; + expect(declare).toBeTypeOf("function"); + }); + + it("refuses a native source as a state member (Q14)", async () => { + const Bad = local("app.bad", { version: 1, params: {}, members: { n: int() } }); + const { host } = await loaded((h) => h.serve(Bad, (_, self) => ({ n: self.writable(1) as never }))); + expect(await rejection(host.get(Bad, {}, context))).toMatchObject({ + message: "app.bad: n: a state member is a JSON watchable", + }); + await host.close(context); + }); +}); + +describe("watching a native source that fails", () => { + it("a watch taken while it is failed opens at its last good value and gets the failure first", async () => { + const { host } = await loaded(serveCount); + const count = await host.registry(Count, context); + await (await host.get(Setter, {}, context)).set(-1, context); + const calls: NativeWatchCall[] = []; + const watch = await count.watch(context); + expect(watch.value).toBe(1); + watch.start(async (call) => void calls.push(call)); + await delivered(); + expect(calls).toEqual([{ kind: "failed", error: new Error("negative: -1") }]); + await host.close(context); + }); + + it("an `on` routine started while its source is failed waits for its next good value, never calls with the stale one", async () => { + const Follower = local("app.follower", { version: 1, params: { id: int() }, members: {} }); + const seen: number[] = []; + const { host } = await loaded((h) => { + serveCount(h); + h.serve(Follower, (_, self) => { + self.on(self.registry(Count), ({ value }) => void seen.push(value)); + return {}; + }); + }); + const setters = await host.get(Setter, {}, context); + await setters.set(-1, context); + // Read: the failure is composed before the follower starts. + void (await host.registry(Count, context)).value; + const follower = await host.get(Follower, { id: 1 }, context); + await flushed(); + expect(seen).toEqual([]); + await setters.set(4, context); + await delivered(); + expect(seen).toEqual([4]); + follower.release(); + await host.close(context); + }); +}); diff --git a/packages/chord/test/next/state/writable.test.ts b/packages/chord/test/next/state/writable.test.ts new file mode 100644 index 000000000..30b27309f --- /dev/null +++ b/packages/chord/test/next/state/writable.test.ts @@ -0,0 +1,210 @@ +import { getEventListeners } from "node:events"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { int, local, method } from "../../../src/next/contracts/index.ts"; +import type { ChordHost } from "../../../src/next/host/host.ts"; +import type { Self } from "../../../src/next/host/index.ts"; +import { ChordObject } from "../../../src/next/host/object.ts"; +import { defineRegistry } from "../../../src/next/registries/index.ts"; +import type { Writable } from "../../../src/next/state/index.ts"; +import type { WritableSource } from "../../../src/next/state/internal.ts"; +import type { NativeWatchCall, WatchEnd } from "../../../src/watch.ts"; +import { delivered, expectCollected, instancesOf, loaded, metered, rejection, restartable } from "../support.ts"; + +const objects = instancesOf(ChordObject, "attempt"); + +const Counter = local("app.counter", { version: 1, params: {}, members: { doubled: int(), set: method(int()) } }); + +/** Serves `Counter` with a writable number, its double as a state member; the writable is handed out to the test. */ +async function counter(options: Parameters[1] = {}) { + let writable: Writable | undefined; + let self: Self | undefined; + const { host, errors } = await loaded((h) => { + h.serve(Counter, (_, made) => { + self = made; + const value = made.writable(1); + writable = value; + return { + doubled: made.derive([value], ([n]) => n * 2), + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }, options); + const reference = await host.get(Counter, {}, context); + return { host, errors, reference, writable: writable!, self: self! }; +} + +describe("writable", () => { + it("a new value (by identity) is a new frame; the same value notifies nobody and costs one comparison", async () => { + const { telemetry, counts } = await metered(); + const { host, reference, writable } = await counter({ telemetry }); + const calls: NativeWatchCall[] = []; + (await writable.watch(context)).start(async (call) => void calls.push(call)); + const doubled: number[] = []; + (await reference.doubled.watch(context)).start( + async (call) => void (call.kind === "value" && doubled.push(call.value)), + ); + const computes = counts()["chord.derive.computes"]; + writable.set(1); + await delivered(); + expect(calls).toEqual([]); + expect(counts()["chord.derive.computes"]).toBe(computes); + writable.set(2); + writable.set(3); + await delivered(); + expect(calls).toEqual([ + { kind: "value", value: 2 }, + { kind: "value", value: 3 }, + ]); + // A JSON watch folds what came while it waited: the derive's last value. + expect(doubled).toEqual([6]); + expect(writable.value).toBe(3); + await host.close(context); + }); + + it("inside a batch, sets mark it there: one frame with the last value, the derives over it computed once", async () => { + const { telemetry, counts } = await metered(); + const { host, reference, writable } = await counter({ telemetry }); + const calls: NativeWatchCall[] = []; + (await writable.watch(context)).start(async (call) => void calls.push(call)); + const doubled: number[] = []; + (await reference.doubled.watch(context)).start( + async (call) => void (call.kind === "value" && doubled.push(call.value)), + ); + const computes = counts()["chord.derive.computes"]!; + (host as unknown as ChordHost).batch.commit(() => { + writable.set(5); + writable.set(6); + }, context); + await delivered(); + expect(calls).toEqual([{ kind: "value", value: 6 }]); + expect(doubled).toEqual([12]); + expect(counts()["chord.derive.computes"]).toBe(computes + 1); + await host.close(context); + }); + + it("refuses a set inside a derive compute or a compose (`invalid/reentrant`)", async () => { + const Echo = defineRegistry("app.echo", { version: 1 }); + let inner: Writable | undefined; + const Bad = local("app.bad", { version: 1, params: {}, members: { bad: int() } }); + const Good = local("app.good", { version: 1, params: {}, members: {} }); + const { host, errors } = await loaded((h) => { + h.serve(Echo, { + compose: (table) => { + inner?.set(table.length + 100); + return table.length; + }, + }); + h.serve(Good, (_, self) => { + inner = self.writable(0); + self.contribute(Echo, inner); + return {}; + }); + h.serve(Bad, (_, self) => { + const own = self.writable(0); + return { + bad: self.derive([own], ([n]) => { + own.set(n + 1); + return n; + }), + }; + }); + }); + expect((await host.registry(Echo, context)).value).toBe(0); + expect(errors.find((error) => error.kind === "registry_failed")).toMatchObject({ + error: { code: "invalid", reason: "reentrant" }, + }); + expect(errors.find((error) => error.kind === "activation_failed")).toMatchObject({ + object: "app.bad", + failure: { kind: "broken", error: { code: "invalid", reason: "reentrant" } }, + }); + await host.close(context); + }); + + it("is the attempt's: `set` after its end throws `released`, its watches end `ended`, its value stays readable", async () => { + let writable: Writable | undefined; + const { host, restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) writable = self.writable(1); + }); + const watch = await writable!.watch(context); + await restart(); + expect(await watch.closed).toEqual({ kind: "ended" } satisfies WatchEnd); + expect(() => writable!.set(2)).toThrow("app.restartable(1): ended"); + expect(writable!.value).toBe(1); + await host.close(context); + }); + + it("is a contribution, a derive source and an `on` source", async () => { + const Values = defineRegistry("app.values", { version: 1 }); + const Source = local("app.source", { version: 1, params: {}, members: { set: method(int()), seen: int() } }); + const seen: number[] = []; + const { host } = await loaded((h) => { + h.serve(Values, { compose: (table) => table.map(({ value }) => value) }); + h.serve(Source, (_, self) => { + const value = self.writable(1); + self.contribute(Values, value); + self.on(value, ({ value: n }) => void seen.push(n)); + return { + seen: self.derive([value], ([n]) => n), + set: async (n: number) => { + value.set(n); + return null; + }, + }; + }); + }); + const source = await host.get(Source, {}, context); + const values = await host.registry(Values, context); + await source.set(2, context); + await delivered(); + expect(values.value).toEqual([2]); + expect(source.seen.value).toBe(2); + expect(seen).toEqual([1, 2]); + await host.close(context); + }); + + it("is collected with its attempt", async () => { + const refs = await (async () => { + const made: WeakRef[] = []; + const { host, restart } = await restartable(objects, (attempt, self) => { + if (attempt === 1) made.push(new WeakRef(self.writable({ big: new Array(1000).fill(0) }))); + }); + await restart(); + await host.close(context); + return made; + })(); + await expectCollected(...refs); + }); + + it("a set of the same value after its attempt ended throws `released` too (Sol, Astra)", async () => { + const { host, writable } = await counter(); + await host.close(context); + expect(() => writable.set(1)).toThrow("app.counter: ended"); + }); + + it("a writable an app keeps after `host.close` keeps no closed host; its observers' abort listeners go (Sol)", async () => { + const signal = new AbortController(); + const { weak, writable } = await (async () => { + const { host, writable } = await counter(); + (writable as WritableSource).observe( + { advance: () => {}, end: () => {} }, + withAbortSignal(signal.signal, context), + ); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(1); + await host.close(context); + expect(getEventListeners(signal.signal, "abort")).toHaveLength(0); + return { weak: new WeakRef(host), writable }; + })(); + await expectCollected(weak); + expect(writable.value).toBe(1); + }); + + it("a late callback's set after the host closed throws `released`", async () => { + const { host, writable } = await counter(); + await host.close(context); + expect(await rejection(Promise.resolve().then(() => writable.set(9)))).toMatchObject({ code: "released" }); + }); +}); diff --git a/packages/chord/test/next/store/store.test.ts b/packages/chord/test/next/store/store.test.ts index e94641317..335cbfd91 100644 --- a/packages/chord/test/next/store/store.test.ts +++ b/packages/chord/test/next/store/store.test.ts @@ -6,19 +6,26 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { int, str } from "../../../src/next/contracts/index.ts"; import { isChordError } from "../../../src/next/data/index.ts"; import { Batch } from "../../../src/next/state/internal.ts"; import { sqliteBackend } from "../../../src/next/store/backend.ts"; -import { changeDocument, createDocument, retireDocument } from "../../../src/next/store/engine.ts"; +import { changeDocument, createDocument, findDocument, retireDocument } from "../../../src/next/store/engine.ts"; import { importStore } from "../../../src/next/store/import.ts"; -import type { StagedWrite } from "../../../src/next/store/step.ts"; +import type { StagedWrite, Step } from "../../../src/next/store/step.ts"; import { type ChordStore, openStore, type Store } from "../../../src/next/store/store.ts"; +import { defineTable } from "../../../src/next/store/table.ts"; +import { stageTable } from "../../../src/next/store/table-writes.ts"; import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; import { exported, faultyStore, holdNextCommit, metered, rejection } from "../support.ts"; const ADDRESS = { namespace: "chord", owner: "app.store", kind: "notes", key: "" }; const BASE = { version: 1, fingerprint: "0000000000000000", value: { n: 0 } }; +/** A table a series creates. */ +const Edits = defineTable("edits", { version: 1, columns: { id: str(), at: int() }, key: ["id"] }); +const TABLE = "app.store:edits"; + /** A store on a fresh `:memory:` database, as Chord's layers see it. */ async function memoryStore(options?: { readonly outbox?: boolean }): Promise { return (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context, options)) as ChordStore; @@ -335,3 +342,238 @@ describe("close", () => { expect(await rejection(store.read(async () => 1, context))).toMatchObject({ code: "released" }); }); }); + +describe("a commit series", () => { + /** A series commit with no check of its own. */ + const unchecked = (writes: readonly StagedWrite[]) => ({ check: async () => {}, writes }); + + it("stores its commits in one transaction, one seq each, each checked against the ones before it", async () => { + const store = await memoryStore({ outbox: true }); + const seen: (number | undefined)[] = []; + const receipts: [number, number, readonly number[]][] = []; + const telemetry = await metered(); + const result = await store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + // No writes: no seq, it stands at the commit before it. + unchecked([]), + { + // The check reads through its step: it sees the first commit's creation. + check: async (step) => void seen.push((await findDocument(step, ADDRESS))?.id), + writes: [createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], + }, + ], + (index, seq, ids) => receipts.push([index, seq, ids]), + telemetry.context, + ); + expect(result).toEqual({ stored: 3 }); + expect(seen).toEqual([2]); + expect(receipts).toEqual([ + [0, 1, [2]], + [1, 1, []], + [2, 2, [3]], + ]); + expect(store.lastSeq).toBe(2); + expect(telemetry.counts()["chord.store.commits"]).toBe(2); + const outbox = await store.read( + (executor) => executor.all<{ seq: number }>("SELECT seq FROM chord_outbox"), + context, + ); + expect(outbox.map((row) => row.seq)).toEqual([1, 2]); + }); + + it("stops at a check that refuses: the commits before it are stored, it and the ones after are not", async () => { + const store = await memoryStore(); + const refusal = new Error("refused"); + const receipts: number[] = []; + const result = await store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + { + check: () => Promise.reject(refusal), + writes: [createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], + }, + unchecked([createDocument({ ...ADDRESS, key: "c" }, BASE, "{}")]), + ], + (index) => receipts.push(index), + context, + ); + expect(result).toEqual({ stored: 1, refused: refusal }); + expect(receipts).toEqual([0]); + expect(store.lastSeq).toBe(1); + const keys = await store.read( + (executor) => executor.all<{ key: string }>("SELECT key FROM chord_documents ORDER BY id"), + context, + ); + expect(keys.map((row) => row.key)).toEqual([""]); + // The store goes on at the next seq. + const { adopted, fence } = recording(); + await store.commit([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")], fence, context); + expect(adopted).toEqual([[2, [3]]]); + }); + + it("rolls the whole series back when a check's statement fails, even if the check throws its own error", async () => { + const store = await memoryStore(); + const own = new Error("own"); + const series = store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + { + check: async (step) => { + await step.get("SELECT * FROM no_such_table").catch(() => {}); + throw own; + }, + writes: [], + }, + ], + () => {}, + context, + ); + expect(await rejection(series)).toBe(own); + expect(store.lastSeq).toBe(0); + // A rollback, not a stop: the store goes on; the id the rolled back creation took is not given back (D3). + expect(await created(store)).toBe(3); + }); + + it("rolls the whole series back when a commit's writes throw, the store going on", async () => { + const store = await memoryStore(); + const receipts: number[] = []; + const error = await rejection( + store.commitSeries( + [unchecked([createDocument(ADDRESS, BASE, "{}")]), unchecked([BROKEN])], + (index) => receipts.push(index), + context, + ), + ); + expect(String(error)).toContain("no such table"); + expect([receipts, store.lastSeq]).toEqual([[], 0]); + expect(await created(store)).toBe(3); + }); + + it("writes the metadata once per series: three statements per delta with its outbox commit, plus one (cost guard)", async () => { + const store = await memoryStore({ outbox: true }); + const id = await created(store); + const telemetry = await metered(); + const delta = (n: number) => { + const ops = [["s", ["n"], n]] as const; + return unchecked([changeDocument(id, n, { kind: "delta", version: 1, ops }, JSON.stringify(ops))]); + }; + await store.commitSeries([delta(1), delta(2), delta(3)], () => {}, telemetry.context); + // A delta committed alone is four statements with an outbox, the metadata among them; a series writes it once. + expect(telemetry.counts()).toEqual({ "chord.store.commits": 3, "chord.store.statements": 3 * 3 + 1 }); + const meta = await store.read( + (executor) => executor.get<{ last_seq: number }>("SELECT last_seq FROM chord_meta"), + context, + ); + expect(meta?.last_seq).toBe(4); + }); + + it.each([ + ["BEGIN IMMEDIATE", undefined], + ["COMMIT", undefined], + ["ROLLBACK", BROKEN], + ] as const)( + "a failed %s is an unknown outcome: nothing is received, the store stops storage_failed", + async (at, write) => { + const { store, fail } = await faultyStore(); + const told: unknown[] = []; + store.attach({ failed: (error) => told.push(error), batch: new Batch(() => {}, undefined) }); + fail(at); + const receipts: number[] = []; + const second = write ?? createDocument({ ...ADDRESS, key: "b" }, BASE, "{}"); + const series = store.commitSeries( + [unchecked([createDocument(ADDRESS, BASE, "{}")]), unchecked([second])], + (index) => receipts.push(index), + context, + ); + expect(await rejection(series)).toMatchObject({ code: "unavailable", reason: "storage_failed" }); + expect([receipts, store.lastSeq, told.length]).toEqual([[], 0, 1]); + const after = store.commitSeries([unchecked([createDocument(ADDRESS, BASE, "{}")])], () => {}, context); + expect(await rejection(after)).toMatchObject({ reason: "storage_failed" }); + }, + ); + + it("lets a later commit see the table records an earlier one changed; a rollback leaves the catalog as it was", async () => { + const store = await memoryStore(); + const { version, record: columns, key, fingerprint } = Edits; + const table = stageTable({ kind: "table", table: TABLE, version, state: "ready", columns, key, fingerprint }); + await rejection(store.commitSeries([unchecked([table]), unchecked([BROKEN])], () => {}, context)); + expect(store.table(TABLE)).toBeUndefined(); + const seen: (number | undefined)[] = []; + const sees: StagedWrite = { + order: "write", + apply: async (step) => { + seen.push(step.table(TABLE)?.version); + return []; + }, + }; + const later = { check: async (step: Step) => void seen.push(step.table(TABLE)?.version), writes: [sees] }; + expect(await store.commitSeries([unchecked([table]), later], () => {}, context)).toEqual({ stored: 2 }); + expect(seen).toEqual([1, 1]); + expect(store.table(TABLE)?.version).toBe(1); + }); + + it("runs a commit queued behind a series after the series' receipts", async () => { + const store = await memoryStore(); + const events: string[] = []; + const series = store.commitSeries( + [ + unchecked([createDocument(ADDRESS, BASE, "{}")]), + unchecked([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")]), + ], + (index, seq) => events.push(`receive ${index} ${seq}`), + context, + ); + const fence = { + admit: () => { + events.push("admit"); + return true; + }, + adopt: (seq: number) => void events.push(`adopt ${seq}`), + }; + const queued = store.commit([createDocument({ ...ADDRESS, key: "c" }, BASE, "{}")], fence, context); + await Promise.all([series, queued]); + expect(events).toEqual(["receive 0 1", "receive 1 2", "admit", "adopt 3"]); + }); + + it("counts only commits with writes against the seq bound: empty ones at Number.MAX_SAFE_INTEGER still stand", async () => { + const [manifest, ...rest] = await exported(await memoryStore()); + const parts = [{ ...manifest!, lastSeq: Number.MAX_SAFE_INTEGER - 1 }, ...rest]; + const store = (await importStore( + sqliteBackend(await openNodeSqliteDatabase(":memory:")), + parts, + context, + )) as ChordStore; + await created(store); + const receipts: [number, number][] = []; + const empty = await store.commitSeries( + [unchecked([]), unchecked([])], + (index, seq) => receipts.push([index, seq]), + context, + ); + expect(empty).toEqual({ stored: 2 }); + expect(receipts).toEqual([ + [0, Number.MAX_SAFE_INTEGER], + [1, Number.MAX_SAFE_INTEGER], + ]); + const writing = store.commitSeries( + [unchecked([createDocument({ ...ADDRESS, key: "b" }, BASE, "{}")])], + () => {}, + context, + ); + expect(await rejection(writing)).toMatchObject({ reason: "storage_failed" }); + }); + + it("stops the store when a receipt throws", async () => { + const store = await memoryStore(); + const series = store.commitSeries( + [unchecked([createDocument(ADDRESS, BASE, "{}")])], + () => { + throw new Error("receipt threw"); + }, + context, + ); + expect(await rejection(series)).toMatchObject({ reason: "storage_failed" }); + expect(await rejection(store.read(async () => 1, context))).toMatchObject({ reason: "storage_failed" }); + }); +}); diff --git a/packages/chord/test/next/store/table-handle.test.ts b/packages/chord/test/next/store/table-handle.test.ts index 0a572d37c..9d147ad83 100644 --- a/packages/chord/test/next/store/table-handle.test.ts +++ b/packages/chord/test/next/store/table-handle.test.ts @@ -10,8 +10,9 @@ import type { StagedWrite } from "../../../src/next/store/step.ts"; import type { ChordStore } from "../../../src/next/store/store.ts"; import { defineTable } from "../../../src/next/store/table.ts"; import { TableChanges } from "../../../src/next/store/table-changes.ts"; +import { evolveTable } from "../../../src/next/store/table-evolve.ts"; import type { TableHandle } from "../../../src/next/store/table-handle.ts"; -import { memoryStore, rejection, tableIn, tableOwner } from "../support.ts"; +import { holdNextCommit, memoryStore, rejection, tableIn, tableOwner } from "../support.ts"; const Edits = defineTable("edits", { version: 1, @@ -134,3 +135,33 @@ describe("unreadable cells", () => { }); }); }); + +describe("the version fence of old handles (R15, slice 7b §5)", () => { + it("refuses reads through a handle older than the stored table: released/version", async () => { + store = await memoryStore(); + const old = await tableIn(store, Edits); + const V2 = evolveTable(Edits, (t) => t.rename("note", "comment")); + await tableIn(store, V2); + const refused = { code: "released", reason: "version" }; + expect(await rejection(old.get({ id: "e0" }, context))).toMatchObject(refused); + expect(await rejection(old.count(query(Edits), context))).toMatchObject(refused); + expect(() => old.assertVersion()).toThrow("the store has version 2"); + }); +}); + +describe("the version fence against a migration in flight (fix round 1, Sol 8)", () => { + it("refuses an old handle's read made while the migration's commit runs, rather than issuing obsolete SQL", async () => { + const V1 = defineTable("race", { version: 1, columns: { id: str(), old: str() }, key: ["id"] }); + const V2 = evolveTable(V1, (t) => t.rename("old", "new")); + store = await memoryStore(); + const old = await tableIn(store, V1); + const held = holdNextCommit(store); + const changing = tableIn(store, V2); + await held.entered; + const reading = rejection(old.get({ id: "x" }, context)); + held.release(); + await changing; + expect(await reading).toMatchObject({ code: "released", reason: "version" }); + expect(() => store.assertUsable()).not.toThrow(); + }); +}); diff --git a/packages/chord/test/next/store/table-migration.test.ts b/packages/chord/test/next/store/table-migration.test.ts index d58bef1b7..e5938015b 100644 --- a/packages/chord/test/next/store/table-migration.test.ts +++ b/packages/chord/test/next/store/table-migration.test.ts @@ -218,18 +218,33 @@ describe("eager migrations", () => { }); }); +/** + * Records the `setTimeout` calls the migration's own code makes (its stack passes through `table-migration.ts`); timers + * of the test runner or other work under load pass through unrecorded (fix round 2: the test watched every timer). + */ +function migrationTimers(): { readonly calls: string[]; restore(): void } { + const calls: string[] = []; + const original = globalThis.setTimeout; + const spy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((...args: Parameters) => { + const stack = new Error().stack ?? ""; + if (stack.includes("src/next/store/table-migration.ts")) calls.push(stack); + return original(...args); + }) as typeof setTimeout); + return { calls, restore: () => spy.mockRestore() }; +} + describe("the chunk yield", () => { it("yields between chunks without Node's setImmediate (Workers, browsers), through a message, not a timer", async () => { await filled(3 * MIGRATION_CHUNK); const saved = globalThis.setImmediate; // Runtimes without it: Cloudflare Workers, browsers. Reflect.deleteProperty(globalThis, "setImmediate"); - const timers = vi.spyOn(globalThis, "setTimeout"); + const timers = migrationTimers(); try { await prepared(EditsV2); - expect(timers).not.toHaveBeenCalled(); + expect(timers.calls).toEqual([]); } finally { - timers.mockRestore(); + timers.restore(); globalThis.setImmediate = saved; } expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); @@ -242,12 +257,12 @@ describe("the chunk yield", () => { setTimeout(() => { fired = true; }, 0); - const timers = vi.spyOn(globalThis, "setTimeout"); + const timers = migrationTimers(); try { await prepared(EditsV2); - expect(timers).not.toHaveBeenCalled(); + expect(timers.calls).toEqual([]); } finally { - timers.mockRestore(); + timers.restore(); } expect(fired).toBe(true); expect(store.table(TABLE)).toMatchObject({ version: 2, state: "ready" }); diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 428dea36f..20732c04f 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -35,9 +35,12 @@ import { createHost, defineFacet, type FacetHost, + type FacetSelf, type Host, type HostError, + type PluginFailure, type RefState, + type RouteHandle, type Self, type StoreSelf, } from "../../src/next/host/index.ts"; @@ -231,7 +234,7 @@ async function restartableOf void, + setup: (host: FacetHost, self: FacetSelf) => void, options: { readonly telemetry?: InMemoryTelemetryContext; readonly context?: Context; @@ -239,12 +242,28 @@ export async function loaded( readonly readyTimeout?: number; readonly store?: Store; } = {}, -): Promise<{ readonly host: Host; readonly errors: HostError[] }> { +): Promise<{ readonly host: Host; readonly errors: HostError[]; readonly failed: readonly PluginFailure[] }> { const errors: HostError[] = []; const { telemetry, closeTimeout, readyTimeout, store } = options; const host = createHost({ telemetry, closeTimeout, readyTimeout, store, onError: (error) => errors.push(error) }); - await host.load([{ name: "@test/app", facet: defineFacet(setup) }], options.context ?? BACKGROUND_CONTEXT); - return { host, errors }; + const facets = [{ name: "@test/app", facet: defineFacet(setup) }]; + const { failed } = await host.load(facets, options.context ?? BACKGROUND_CONTEXT); + return { host, errors, failed }; +} + +/** + * A host with background errors collected, loaded with one facet per plugin, in the order given (`setups` by plugin + * name). + */ +export async function loadedPlugins( + setups: Readonly void>>, + options: { readonly telemetry?: InMemoryTelemetryContext; readonly readyTimeout?: number } = {}, +): Promise<{ readonly host: Host; readonly errors: HostError[]; readonly failed: readonly PluginFailure[] }> { + const errors: HostError[] = []; + const host = createHost({ ...options, onError: (error) => errors.push(error) }); + const facets = Object.entries(setups).map(([name, setup]) => ({ name, facet: defineFacet(setup) })); + const { failed } = await host.load(facets, BACKGROUND_CONTEXT); + return { host, errors, failed }; } /** A store on a fresh SQLite `:memory:` database, as Chord's layers see it. */ @@ -385,6 +404,7 @@ export async function linked( const errors = { server: [] as HostError[], client: [] as HostError[] }; const { telemetry, readyTimeout } = options; const server = createHost({ + address: Server(), principal: { kind: "server" }, telemetry, readyTimeout, @@ -531,6 +551,7 @@ export async function rawLinked( ): Promise<{ readonly server: Host; readonly peer: RawPeer; readonly errors: HostError[] }> { const errors: HostError[] = []; const server = createHost({ + address: Server(), principal: { kind: "server" }, readyTimeout: options.readyTimeout, compression: options.compression, @@ -564,8 +585,19 @@ const NOTES_VARIANTS = { reader: { state: { notes: arr(obj({ text: str() })), count: int() }, methods: { wait: method() } }, }; +/** The host kind of the test servers, where every test type but `PlacedNotes` lives. */ +export const Server = hostKind("server"); + +/** `Server()` as a hold names it on the wire: the address of every test type served on the server. */ +export const SERVER_ADDRESS = { kind: "server", params: {} }; + /** The remote type of the link tests: notes per session, an editor variant for owners, a reader one for guests. */ -export const Notes = remote("app.notes", { version: 1, params: { session: str() }, variants: NOTES_VARIANTS }); +export const Notes = remote("app.notes", { + version: 1, + params: { session: str() }, + hostAddress: () => Server(), + variants: NOTES_VARIANTS, +}); /** The kind of the relay tests' workers: one per session. */ export const Session = hostKind("session", { id: str() }); @@ -574,7 +606,7 @@ export const Session = hostKind("session", { id: str() }); export const PlacedNotes = remote("app.placed-notes", { version: 1, params: { session: str() }, - placement: ({ session }) => Session({ id: session }), + hostAddress: ({ session }) => Session({ id: session }), variants: NOTES_VARIANTS, }); @@ -614,7 +646,7 @@ export function serveNotes( }, }; }); - // The two types differ in their name and placement only. + // The two types differ in their name and host address only. host.serve(Type as typeof Notes, ({ session }, principal, self) => { probe.factories.push(`${session} ${principal.kind}`); const store = self.get(NoteStore, { session }); @@ -739,6 +771,8 @@ export type Resolved = { export type Relayed = { readonly tab: Host; readonly server: Host; + /** The server's route to `Session`'s addresses. */ + readonly route: RouteHandle; /** The worker of each resolved session address, the latest one per key. */ readonly workers: Map>; /** The server's end of each resolved route link, in order. */ @@ -791,6 +825,7 @@ export async function relayed( const resolves: string[] = []; const { revokedBy, compression = {} } = options; const server = createHost({ + address: Server(), principal: { kind: "server" }, revokedBy: options.serverRevokedBy ?? revokedBy, compression: compression.server, @@ -819,7 +854,7 @@ export async function relayed( return { link: a, principal, trusted: options.trusted, codec }; }; const resolve = options.resolve ?? (async (_address, made) => made()); - server.route( + const route = server.route( Session, { authorize: options.authorize ?? (() => true), @@ -838,7 +873,7 @@ export async function relayed( const accepted = { principal: options.tab ?? { kind: "owner" }, codec: tabCodec, limits: options.limits } as const; server.accept(a, accepted, context); tab.connect(b, { principal: { kind: "server" }, codec: tabCodec }, context); - return { tab, server, workers, routeLinks, tabLink: a, resolves, errors }; + return { tab, server, route, workers, routeLinks, tabLink: a, resolves, errors }; } /** diff --git a/packages/chord/test/next/surfaces/active-theme.test.ts b/packages/chord/test/next/surfaces/active-theme.test.ts new file mode 100644 index 000000000..540306f98 --- /dev/null +++ b/packages/chord/test/next/surfaces/active-theme.test.ts @@ -0,0 +1,123 @@ +// Tests of the active theme of a client (slice 11 §4b) and of theme choices on the server: a client follows the +// `Themes` registry, its principal's `ThemeChoice` and its mode; choices are kept per principal across restarts. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + createHost, + defineFacet, + followActiveTheme, + hostKind, + memoryLinkPair, + openStore, + sqliteBackend, + type Theme, + ThemeChoice, + Themes, + themeChoiceFacet, + themesFacet, +} from "../../../src/next/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +type TestPrincipal = { readonly kind: "server" } | { readonly kind: "user"; readonly id: string }; + +const COLORS = { + accent: "#111111", + bg: "#000000", + fg: "#ffffff", + muted: "#888888", + error: "#f00", + warning: "#ff0", + success: "#0f0", +}; + +const theme = (id: string, mode: Theme["mode"]): Theme => ({ id, name: id, mode, colors: COLORS }); + +/** A server with theme choices, and a client of `user` with themes `themes` contributed by a plugin. */ +async function setup(user: string, themes: readonly Theme[]) { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); + const server = createHost({ + address: hostKind("server")(), + principal: { kind: "server" }, + store, + onError: () => {}, + }); + await server.load([{ name: "chord.themes.choice", facet: themeChoiceFacet() }], context); + const client = createHost({ onError: () => {} }); + await client.load( + [ + { name: "chord.themes", facet: themesFacet() }, + ...themes.map((each, index) => ({ + name: `@test/theme-${index}`, + facet: defineFacet((_host, self) => self.contribute(Themes, each)), + })), + ], + context, + ); + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user", id: user } }, context); + client.connect(mine, { principal: { kind: "server" } }, context); + return { server, client }; +} + +/** Waits until `check` holds. */ +async function until(check: () => boolean): Promise { + for (let index = 0; index < 200 && !check(); index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(check()).toBe(true); +} + +describe("following the active theme", () => { + it("follows the registry, the principal's pick and the mode", async () => { + const { server, client } = await setup("ada", [ + theme("night", "dark"), + theme("dusk", "dark"), + theme("paper", "light"), + ]); + const seen: string[] = []; + const follower = await followActiveTheme(client, "dark", (active) => seen.push(active.id), context); + await until(() => seen.at(-1) === "night"); + const choice = await client.get(ThemeChoice, {}, context); + await choice.choose({ dark: "dusk", light: null }, context); + await until(() => seen.at(-1) === "dusk"); + follower.setMode("light"); + expect(seen.at(-1)).toBe("paper"); + follower.stop(); + choice.release(); + await client.close(context); + await server.close(context); + }); +}); + +describe("theme choices", () => { + it("refuse an id over 128 characters (Opus M4, Fable L7)", async () => { + const { server, client } = await setup("ada", []); + const choice = await client.get(ThemeChoice, {}, context); + await expect(choice.choose({ dark: "x".repeat(129), light: null }, context)).rejects.toMatchObject({ + code: "invalid", + }); + await choice.choose({ dark: "x".repeat(128), light: null }, context); + choice.release(); + await client.close(context); + await server.close(context); + }); + + it("are kept per principal", async () => { + const { server, client } = await setup("ada", []); + const choice = await client.get(ThemeChoice, {}, context); + expect(choice.pick.value).toEqual({ dark: null, light: null }); + await choice.choose({ dark: "dusk", light: "paper" }, context); + await until(() => choice.pick.value.dark === "dusk"); + const other = createHost({ onError: () => {} }); + await other.load([], context); + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user", id: "grace" } }, context); + other.connect(mine, { principal: { kind: "server" } }, context); + const graces = await other.get(ThemeChoice, {}, context); + expect(graces.pick.value).toEqual({ dark: null, light: null }); + choice.release(); + graces.release(); + await other.close(context); + await client.close(context); + await server.close(context); + }); +}); diff --git a/packages/chord/test/next/surfaces/client.test.ts b/packages/chord/test/next/surfaces/client.test.ts new file mode 100644 index 000000000..82017c549 --- /dev/null +++ b/packages/chord/test/next/surfaces/client.test.ts @@ -0,0 +1,179 @@ +// Tests of a client host with its plugins (slice 11 §2, §4, decisions "Slice 11, P4" to "P6"), over an in-memory link +// to a server serving the scripts object: the initial scripts loaded with the core facets, a new set applied in place +// (only changed scripts fetched; a failed one keeps its old facet), a new base or an outdated server ending the client. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { + constant, + createHost, + defineFacet, + hostKind, + local, + memoryLinkPair, + type RegisteredScript, + type ScriptSet, + type SocketEvents, + startClient, + str, + WebScripts, +} from "../../../src/next/index.ts"; +import { publishTo, scriptsFacet } from "../../../src/next-node/surfaces/current.ts"; + +type TestPrincipal = { readonly kind: "server" } | { readonly kind: "user"; readonly id: string }; + +const Greeting = local("test.greeting", { version: 1, params: {}, members: { text: str() } }); + +/** A registered script of plugin `plugin` (digest `digest`) serving `Greeting` with `text`; `broken` throws instead. */ +function script(plugin: string, digest: string, text: string): RegisteredScript { + return { + plugin, + digest, + run: (require, module) => { + if (text === "broken") throw new Error("broken plugin"); + const chord = require("@test/chord") as typeof chordNext; + const facet = chord.defineFacet((host) => host.serve(Greeting, () => ({ text: chord.constant(text) }))); + Object.assign(module, { exports: { default: facet } }); + }, + }; +} + +/** The set of `scripts` under base `base`. */ +function setOf(base: string, scripts: readonly RegisteredScript[]): ScriptSet { + return { + base, + combined: "/combined.js", + plugins: scripts.map(({ plugin, digest }) => ({ name: plugin, digest, url: `/${plugin}.${digest}.js` })), + }; +} + +/** A server serving `WebScripts` with `initial`, and a client started against it with `scripts` available to fetch. */ +async function setup(initial: readonly RegisteredScript[], available: readonly RegisteredScript[] = []) { + const server = createHost({ + address: hostKind("server")(), + principal: { kind: "server" }, + onError: () => {}, + }); + await server.load( + [{ name: "chord.web", facet: scriptsFacet("web", { set: setOf("B1", initial) }) }], + context, + ); + const fetched: string[] = []; + const errors: chordNext.ClientError[] = []; + const outdated: string[] = []; + const sockets: { events: SocketEvents; link: chordNext.Link }[] = []; + let serverBase = "B1"; + const client = await startClient({ + scripts: WebScripts, + base: "B1", + initial, + modules: { "@test/chord": chordNext }, + core: [ + { name: "@test/core", facet: defineFacet((host) => host.serve(Echo, () => ({ text: constant("core") }))) }, + ], + open: (events) => { + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user", id: "u" } }, context); + sockets.push({ events, link: mine }); + queueMicrotask(() => events.opened()); + return mine; + }, + fetchScript: async (entry) => { + fetched.push(entry.url); + const found = available.find(({ plugin, digest }) => plugin === entry.name && digest === entry.digest); + if (found === undefined) throw new Error(`no ${entry.url}`); + return found; + }, + serverBase: async () => serverBase, + outdated: () => outdated.push("outdated"), + environment: { + now: () => Date.now(), + after: (ms, callback) => { + const timer = setTimeout(callback, ms); + return () => clearTimeout(timer); + }, + online: () => true, + onWake: () => () => {}, + }, + onError: (error) => errors.push(error), + }); + return { + server, + client, + fetched, + errors, + outdated, + sockets, + setServerBase: (base: string) => { + serverBase = base; + }, + }; +} + +const Echo = local("test.echo", { version: 1, params: {}, members: { text: str() } }); + +/** Waits until `check` holds. */ +async function until(check: () => boolean): Promise { + for (let index = 0; index < 200 && !check(); index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(check()).toBe(true); +} + +describe("loading", () => { + it("loads the initial scripts with the core facets", async () => { + const { client, fetched, server } = await setup([script("@test/a", "d1", "hello")]); + expect((await client.host.get(Greeting, {}, context)).text.value).toBe("hello"); + expect((await client.host.get(Echo, {}, context)).text.value).toBe("core"); + expect(fetched).toEqual([]); + await client.stop(context); + await server.close(context); + }); +}); + +describe("applying a new set in place", () => { + it("fetches only changed scripts, reloads the host, keeps an unchanged plugin's facet", async () => { + const a1 = script("@test/a", "d1", "hello"); + const a2 = script("@test/a", "d2", "hello again"); + const { client, fetched, server } = await setup([a1], [a2]); + const greeting = await client.host.get(Greeting, {}, context); + await publishTo(server, "web", setOf("B1", [a2]), context); + await until(() => greeting.text.value === "hello again"); + expect(fetched).toEqual(["/@test/a.d2.js"]); + await client.stop(context); + await server.close(context); + }); + + it("keeps a plugin's old facet when its new script fails, and reports it", async () => { + const a1 = script("@test/a", "d1", "hello"); + const broken = script("@test/a", "d2", "broken"); + const { client, server, errors } = await setup([a1], [broken]); + const greeting = await client.host.get(Greeting, {}, context); + await publishTo(server, "web", setOf("B1", [broken]), context); + await until(() => errors.length > 0); + expect(errors[0]).toMatchObject({ kind: "plugins", failed: [{ plugin: "@test/a" }] }); + expect(greeting.text.value).toBe("hello"); + await client.stop(context); + await server.close(context); + }); +}); + +describe("outdated clients", () => { + it("calls outdated for a set with another base", async () => { + const { client, server, outdated } = await setup([]); + await publishTo(server, "web", setOf("B2", []), context); + await until(() => outdated.length === 1); + await client.stop(context); + await server.close(context); + }); + + it("asks the server's base before reconnecting: another one calls outdated and stops", async () => { + const { client, server, outdated, sockets, setServerBase } = await setup([]); + setServerBase("B2"); + sockets[0]!.link.close(); + await until(() => outdated.length === 1); + await new Promise((resolve) => setTimeout(resolve, 400)); + expect(sockets).toHaveLength(1); + await client.stop(context); + await server.close(context); + }); +}); diff --git a/packages/chord/test/next/surfaces/connection.test.ts b/packages/chord/test/next/surfaces/connection.test.ts new file mode 100644 index 000000000..68171f865 --- /dev/null +++ b/packages/chord/test/next/surfaces/connection.test.ts @@ -0,0 +1,214 @@ +// Tests of the connection loop (slice 11 §2, decision "Slice 11, P6"): reconnect pauses (250 ms doubling to 5 s, reset +// by a stable connection, the longest while offline), wakes (a pending retry at once; an open socket silent past the +// server's pings replaced), the connect timeout, `canRetry` stopping it, and the states it reports. + +import { describe, expect, it } from "vitest"; +import { + CONNECT_TIMEOUT_MS, + type ConnectionState, + keepConnected, + type LoopEnvironment, + type SocketEvents, + STABLE_MS, + STALE_MS, +} from "../../../src/next/surfaces/connection.ts"; + +/** A clock and timers the test advances, wakes it sends, and the sockets the loop opened. */ +class FakeWorld { + now = 0; + online = true; + readonly #timers: { at: number; callback: () => void; cancelled: boolean }[] = []; + #wake: (() => void) | undefined; + readonly sockets: FakeSocket[] = []; + readonly opens: number[] = []; + + readonly environment: LoopEnvironment = { + now: () => this.now, + after: (ms, callback) => { + const timer = { at: this.now + ms, callback, cancelled: false }; + this.#timers.push(timer); + return () => { + timer.cancelled = true; + }; + }, + online: () => this.online, + onWake: (listener) => { + this.#wake = listener; + return () => { + this.#wake = undefined; + }; + }, + }; + + open = (events: SocketEvents): FakeSocket => { + const socket = new FakeSocket(events); + this.sockets.push(socket); + this.opens.push(this.now); + return socket; + }; + + /** Advances the clock by `ms`, running due timers in order; settles promises after each. */ + async advance(ms: number): Promise { + const end = this.now + ms; + for (;;) { + await settled(); + const due = this.#timers.filter(({ at, cancelled }) => !cancelled && at <= end).sort((a, b) => a.at - b.at)[0]; + if (due === undefined) break; + this.now = due.at; + due.cancelled = true; + due.callback(); + } + this.now = end; + await settled(); + } + + wake(): void { + this.#wake?.(); + } + + get last(): FakeSocket { + return this.sockets.at(-1)!; + } +} + +/** One socket and its link: the test opens, feeds and closes it. */ +class FakeSocket { + readonly events: SocketEvents; + closed = false; + #onClose: (() => void) | undefined; + + constructor(events: SocketEvents) { + this.events = events; + } + + onClose(listener: () => void): void { + this.#onClose = listener; + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.#onClose?.(); + } +} + +async function settled(): Promise { + for (let index = 0; index < 5; index++) await Promise.resolve(); +} + +/** A loop in `world` recording its states; `retry` answers `canRetry`. */ +function loop(world: FakeWorld, retry: () => Promise = async () => true) { + const states: ConnectionState[] = []; + const handle = keepConnected({ + open: world.open, + onState: (state) => states.push(state), + canRetry: retry, + environment: world.environment, + }); + return { states, handle }; +} + +describe("reconnecting", () => { + it("reconnects after a growing pause, 250 ms doubling to 5 s, and reports its states", async () => { + const world = new FakeWorld(); + const { states } = loop(world); + const pauses: number[] = []; + for (let attempt = 0; attempt < 7; attempt++) { + const closedAt = world.now; + world.last.close(); + await world.advance(5000); + pauses.push(world.opens.at(-1)! - closedAt); + } + expect(pauses).toEqual([250, 500, 1000, 2000, 4000, 5000, 5000]); + expect(states.slice(0, 4)).toEqual(["connecting", "waiting", "connecting", "waiting"]); + }); + + it("starts the pauses again after a connection that stayed open 10 s", async () => { + const world = new FakeWorld(); + const { states } = loop(world); + for (let attempt = 0; attempt < 3; attempt++) { + world.last.close(); + await world.advance(5000); + } + world.last.events.opened(); + expect(states.at(-1)).toBe("connected"); + await world.advance(STABLE_MS); + const before = world.now; + world.last.close(); + await world.advance(5000); + expect(world.opens.at(-1)! - before).toBe(250); + }); + + it("waits the longest pause while offline, and retries at once on a wake", async () => { + const world = new FakeWorld(); + loop(world); + world.online = false; + world.last.close(); + await world.advance(1000); + expect(world.sockets).toHaveLength(1); + world.wake(); + await settled(); + expect(world.sockets).toHaveLength(2); + }); + + it("gives up a socket not open after 20 s", async () => { + const world = new FakeWorld(); + loop(world); + await world.advance(CONNECT_TIMEOUT_MS - 1); + expect(world.sockets[0]!.closed).toBe(false); + await world.advance(1); + expect(world.sockets[0]!.closed).toBe(true); + }); + + it("stops when canRetry says no", async () => { + const world = new FakeWorld(); + const { states } = loop(world, async () => false); + world.last.close(); + await world.advance(60_000); + expect(world.sockets).toHaveLength(1); + expect(states.at(-1)).toBe("waiting"); + }); + + it("retries at once when woken while canRetry is deciding", async () => { + const world = new FakeWorld(); + let answer!: (retry: boolean) => void; + loop( + world, + () => + new Promise((resolve) => { + answer = resolve; + }), + ); + world.last.close(); + world.wake(); + answer(true); + await world.advance(0); + expect(world.sockets).toHaveLength(2); + }); +}); + +describe("a silent socket", () => { + it("is replaced on a wake once its next ping is overdue; one with recent frames stays", async () => { + const world = new FakeWorld(); + loop(world); + world.last.events.opened(); + await world.advance(10_000); + world.last.events.received(); + world.wake(); + await world.advance(STALE_MS); + expect(world.sockets[0]!.closed).toBe(false); + world.wake(); + await world.advance(STALE_MS + 1000); + expect(world.sockets[0]!.closed).toBe(true); + }); + + it("closes the link at stop and opens no other", async () => { + const world = new FakeWorld(); + const { handle } = loop(world); + world.last.events.opened(); + handle.stop(); + expect(world.last.closed).toBe(true); + await world.advance(60_000); + expect(world.sockets).toHaveLength(1); + }); +}); diff --git a/packages/chord/test/next/surfaces/registered.test.ts b/packages/chord/test/next/surfaces/registered.test.ts new file mode 100644 index 000000000..1ae367fcd --- /dev/null +++ b/packages/chord/test/next/surfaces/registered.test.ts @@ -0,0 +1,57 @@ +// Tests of scripts that register themselves (slice 11 §1, §2): the served text calls `chordFacet` with the plugin, the +// digest and the build's function; evaluating it answers only the client's modules (an ES module namespace as a +// CommonJS import sees it) and refuses a default export that is no facet. + +import { Script } from "node:vm"; +import { describe, expect, it } from "vitest"; +import * as chordNext from "../../../src/next/index.ts"; +import { + evaluateRegistered, + type FacetFunction, + REGISTER, + type RegisteredScript, + registeredSource, +} from "../../../src/next/surfaces/registered.ts"; + +/** Runs a registered script's text and returns what it registered. */ +function run(text: string): RegisteredScript[] { + const registered: RegisteredScript[] = []; + (globalThis as Record)[REGISTER] = (plugin: string, digest: string, f: FacetFunction) => + registered.push({ plugin, digest, run: f }); + new Script(text).runInThisContext(); + delete (globalThis as Record)[REGISTER]; + return registered; +} + +/** A built script's function expression, as esbuild wraps a facet: it requires `specifier`, exports a facet. */ +const BUILT = (specifier: string) => + `(function (require, module, exports) { +const chord = require(${JSON.stringify(specifier)}); +module.exports = { default: chord.defineFacet(() => {}) }; +//# sourceMappingURL=data:application/json;base64,e30= +})`; + +describe("registered scripts", () => { + it("register their plugin, digest and function, the source map comment kept inside the call", () => { + const text = registeredSource("@acme/notes", "abc123", BUILT("@acme/app/v1")); + expect(text.startsWith('chordFacet("@acme/notes","abc123",(function (require, module, exports) {')).toBe(true); + const [script] = run(text); + expect([script!.plugin, script!.digest]).toEqual(["@acme/notes", "abc123"]); + const facet = evaluateRegistered(script!, { "@acme/app/v1": chordNext }); + expect(typeof facet.setup).toBe("function"); + }); + + it("answer only the client's modules, naming the plugin", () => { + const [script] = run(registeredSource("@acme/notes", "d", BUILT("node:fs"))); + expect(() => evaluateRegistered(script!, { "@acme/app/v1": chordNext })).toThrow( + 'plugin "@acme/notes": "node:fs" is not one of this client\'s host packages', + ); + }); + + it("refuse a default export that is no facet", () => { + const [script] = run( + registeredSource("@acme/bad", "d", "(function (require, module) { module.exports = { default: 42 }; })"), + ); + expect(() => evaluateRegistered(script!, {})).toThrow('plugin "@acme/bad": its default export is not a facet'); + }); +}); diff --git a/packages/chord/test/next/surfaces/themes.test.ts b/packages/chord/test/next/surfaces/themes.test.ts new file mode 100644 index 000000000..2731b21c1 --- /dev/null +++ b/packages/chord/test/next/surfaces/themes.test.ts @@ -0,0 +1,70 @@ +// Tests of themes (slice 11 §4b): the active theme for a mode and a pick, CSS variables of a theme, and terminal +// colours by role (true colour, the nearest of 256, `tui` overrides first, the fallback for a missing role). + +import { describe, expect, it } from "vitest"; +import { + activeTheme, + FALLBACK_THEMES, + type Theme, + terminalColor, + themeVariables, +} from "../../../src/next/surfaces/themes.ts"; + +const COLORS = { + accent: "#e8590c", + bg: "#111317", + fg: "#e8e8e8", + muted: "#8a8f98", + error: "#ff0000", + warning: "#ffaa00", + success: "#00ff00", +}; + +const SOLARIZED: Theme = { id: "solarized", name: "Solarized", mode: "dark", colors: { ...COLORS, link: "#268bd2" } }; + +const PAPER: Theme = { id: "paper", name: "Paper", mode: "light", colors: COLORS, web: { radius: "10px" } }; + +const NIGHT: Theme = { id: "night", name: "Night", mode: "dark", colors: COLORS, tui: { accent: "208" } }; + +describe("the active theme", () => { + it("is the picked one of the mode, else the first of the mode, else the fallback", () => { + const themes = [SOLARIZED, PAPER, NIGHT]; + expect(activeTheme(themes, { dark: "night", light: null }, "dark").id).toBe("night"); + expect(activeTheme(themes, { dark: null, light: null }, "dark").id).toBe("solarized"); + expect(activeTheme(themes, { dark: "paper", light: null }, "dark").id).toBe("solarized"); + expect(activeTheme(themes, { dark: null, light: "gone" }, "light").id).toBe("paper"); + expect(activeTheme([SOLARIZED], { dark: null, light: null }, "light")).toBe(FALLBACK_THEMES.light); + }); +}); + +describe("CSS variables", () => { + it("gives every role and every web extra a custom property", () => { + expect(themeVariables(SOLARIZED)).toMatchObject({ + "--accent": "#e8590c", + "--bg": "#111317", + "--link": "#268bd2", + }); + expect(themeVariables(PAPER)["--radius"]).toBe("10px"); + }); +}); + +describe("terminal colours", () => { + it("gives true colour, or the nearest of 256, for a role", () => { + expect(terminalColor(SOLARIZED, "accent", "truecolor", "fg")).toBe("38;2;232;89;12"); + expect(terminalColor(SOLARIZED, "bg", "truecolor", "bg")).toBe("48;2;17;19;23"); + expect(terminalColor(SOLARIZED, "error", "256", "fg")).toBe("38;5;196"); + expect(terminalColor(SOLARIZED, "fg", "256", "fg")).toBe("38;5;254"); + expect(terminalColor(SOLARIZED, "bg", "256", "bg")).toBe("48;5;233"); + }); + + it("takes the theme's tui override first, the fallback for a missing role, the default for an unknown one", () => { + expect(terminalColor(NIGHT, "accent", "truecolor", "fg")).toBe("38;5;208"); + const partial = { ...SOLARIZED, colors: { ...COLORS, muted: "not a colour" } }; + expect(terminalColor(partial, "muted", "truecolor", "fg")).toBe("39"); + expect(terminalColor(SOLARIZED, "nobody", "256", "bg")).toBe("49"); + const fallback = FALLBACK_THEMES.dark.colors.accent.slice(1); + const rgb = [0, 2, 4].map((at) => Number.parseInt(fallback.slice(at, at + 2), 16)).join(";"); + const missing = { ...SOLARIZED, colors: { ...COLORS, accent: undefined as unknown as string } }; + expect(terminalColor(missing, "accent", "truecolor", "fg")).toBe(`38;2;${rgb}`); + }); +}); diff --git a/packages/chord/test/next/tui/frames.test.ts b/packages/chord/test/next/tui/frames.test.ts new file mode 100644 index 000000000..7d63caaf9 --- /dev/null +++ b/packages/chord/test/next/tui/frames.test.ts @@ -0,0 +1,26 @@ +// Tests of frames for the terminal (slice 11 §4; fix round 1, Fable L9): a write over the launcher's 64 KiB arrives +// as frames each under it, cut only between escape sequences and surrogate pairs, the filter keeping every one whole. + +import { describe, expect, it } from "vitest"; +import { filterOutput, MAX_FRAME } from "../../../src/next-node/tui/filter.ts"; +import { FRAME_UNITS, splitFrames } from "../../../src/next-tui/frames.ts"; + +describe("splitting frames", () => { + it("keeps a small write whole", () => { + expect(splitFrames("hello")).toEqual(["hello"]); + }); + + it("cuts a large true-colour paint between sequences, each frame under 64 KiB and passing the filter whole", () => { + let paint = ""; + for (let cell = 0; cell < 20_000; cell++) + paint += `\x1b[38;2;${cell % 256};12;200m${cell % 7 === 0 ? "😀" : "é"}`; + const frames = splitFrames(paint); + expect(frames.length).toBeGreaterThan(1); + expect(frames.join("")).toBe(paint); + for (const frame of frames) { + expect(frame.length).toBeLessThanOrEqual(FRAME_UNITS); + expect(Buffer.byteLength(frame)).toBeLessThanOrEqual(MAX_FRAME); + expect(filterOutput(frame)).toBe(frame); + } + }); +}); diff --git a/packages/chord/test/next/tui/scope.test.ts b/packages/chord/test/next/tui/scope.test.ts new file mode 100644 index 000000000..43375c862 --- /dev/null +++ b/packages/chord/test/next/tui/scope.test.ts @@ -0,0 +1,80 @@ +// Tests of `TuiScope` (slice 11 §4, decision "Slice 11, P9"): references held for a component, watchables and +// registries followed, one `changed` callback for every change, and everything released at `dispose`. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet, defineRegistry, local, method, str } from "../../../src/next/index.ts"; +import { TuiScope } from "../../../src/next-tui/scope.ts"; + +const Notes = local("test.notes", { version: 1, params: {}, members: { text: str(), write: method(str()) } }); + +const Panels = defineRegistry("test.panels", { version: 1 }); + +/** A host serving `Notes` (a writable text) and `Panels` with one contribution. */ +async function host() { + const created = createHost({ onError: () => {} }); + await created.load( + [ + { + name: "@test/notes", + facet: defineFacet((h, self) => { + h.serve(Notes, (_params, object) => { + const text = object.writable("first"); + return { + text: object.derive([text], ([value]) => value), + write: async (next) => { + text.set(next); + return null; + }, + }; + }); + h.serve(Panels, { compose: (table) => table.map(({ value }) => value) }); + self.contribute(Panels, "notes"); + }), + }, + ], + context, + ); + return created; +} + +/** Waits until `check` holds. */ +async function until(check: () => boolean): Promise { + for (let index = 0; index < 200 && !check(); index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(check()).toBe(true); +} + +describe("a component's scope", () => { + it("holds, follows and calls changed on every change; releases everything at dispose", async () => { + const h = await host(); + let changes = 0; + const scope = new TuiScope(h, () => changes++); + const notes = scope.hold(Notes, {}); + const text = scope.watch(notes.text); + const panels = scope.registry(Panels); + await until(() => text() === "first" && panels()?.[0] === "notes"); + const before = changes; + await notes.write("second", context); + await until(() => text() === "second" && changes > before); + scope.dispose(); + expect(notes.state.value.kind).toBe("ended"); + const after = changes; + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(changes).toBe(after); + await h.close(context); + }); + + it("reads undefined from a watchable without a value, never throwing", async () => { + const h = await host(); + const scope = new TuiScope(h, () => {}); + const read = scope.watch({ + get value(): string { + throw new Error("not ready"); + }, + watch: () => Promise.reject(new Error("not ready")), + }); + expect(read()).toBeUndefined(); + scope.dispose(); + await h.close(context); + }); +}); diff --git a/packages/chord/test/output.test.ts b/packages/chord/test/output.test.ts index 00e7d647e..b35a62bf6 100644 --- a/packages/chord/test/output.test.ts +++ b/packages/chord/test/output.test.ts @@ -57,6 +57,18 @@ describe("tool output bounds", () => { expect(sanitizeOutput("a\0b\tc\nd\re\u0007f\ufff9g\ufffbh😀")).toBe("ab\tc\ndefgh😀"); }); + // #11604 + it("removes whole ANSI escape sequences, not only their ESC bytes", () => { + expect(sanitizeOutput("\x1b[31mERROR\x1b[0m\n")).toBe("ERROR\n"); + expect(sanitizeOutput("\x1b[1;38;5;208mx\x1b[?25l\x1b[2K\x1b(By\x1b7z")).toBe("xyz"); + expect(sanitizeOutput("\x1b]8;;https://pi.dev\x07link\x1b]8;;\x1b\\ \x1b]0;title\x07ok")).toBe("link ok"); + const output = new OutputBuffer({ retain: "tail", maxBytes: 1024, maxLines: 100 }); + output.push("\x1b[3"); + output.push("1mERROR\x1b[0"); + output.push("m\n"); + expect(output.snapshot().text).toBe("ERROR\n"); + }); + it("keeps output within the limits unchanged", () => { expect(bound("a\nb\n", head(2))).toEqual({ kept: "a\nb\n", droppedBytes: 0, droppedLines: 0 }); expect(bound("a\nb", tail(2))).toEqual({ kept: "a\nb", droppedBytes: 0, droppedLines: 0 }); diff --git a/packages/chord/test/placement.test.ts b/packages/chord/test/placement.test.ts index 869bfa655..3e3a7bd8e 100644 --- a/packages/chord/test/placement.test.ts +++ b/packages/chord/test/placement.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, opt, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { bind, @@ -22,17 +22,17 @@ const tick = () => new Promise((resolve) => setImmediate(resolve)); /** Lives on `session:` + its input's `session`. */ const Note = remote("t.note", { version: 1, - input: Type.Object({ session: Type.String(), id: Type.String() }), + input: obj({ session: str(), id: str() }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); /** Lives on `server`. */ const Catalog = remote("t.catalog", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); /** A server with two session hosts as children; every host serves what it may and reports its name. */ @@ -126,9 +126,9 @@ describe("placement", () => { it("the host comes from the normalized input: unknown fields and defaults do not change it", async () => { const Doc = remote("t.doc-default", { version: 1, - input: Type.Object({ session: Type.String({ default: "S1" }) }), + input: obj({ session: str({ default: "S1" }) }, { open: true }), host: hostFamily("session:", "session"), - variants: { main: { methods: { where: method(undefined, Type.String()) } } }, + variants: { main: { methods: { where: method(undefined, str()) } } }, }); const { server, s1, close } = cluster(); s1.serve(Doc, () => ({ variant: "main", where: () => "session:S1" })); @@ -143,7 +143,7 @@ describe("placement", () => { const at = (host: string) => remote("t.moved", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(host), variants: { main: { methods: { ping: method() } } }, }); @@ -155,7 +155,7 @@ describe("placement", () => { api.serve( remote("t.moved", { version, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed(place), variants: { main: { methods: { ping: method() } } }, }), @@ -170,7 +170,7 @@ describe("placement", () => { // A new version that `accepts` version 1 but lives elsewhere: no adapter makes version 1 consumers bind. const moved = remote("t.moved", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), variants: { main: { methods: { ping: method() } } }, }); @@ -182,7 +182,7 @@ describe("placement", () => { }); it("types: the family's field is a required string of the input, and remote contracts need a host", () => { - const input = Type.Object({ session: Type.String(), count: Type.Number(), note: Type.Optional(Type.String()) }); + const input = obj({ session: str(), count: num(), note: opt(str()) }, { open: true }); const variants = { main: {} }; remote("t.ok", { version: 1, input, host: hostFamily("session:", "session"), variants }); // @ts-expect-error: no such field diff --git a/packages/chord/test/principals.test.ts b/packages/chord/test/principals.test.ts index 4aef22cb3..aab011dab 100644 --- a/packages/chord/test/principals.test.ts +++ b/packages/chord/test/principals.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -26,8 +26,8 @@ const principalKey = (principal: Principal): string => const Who = local("t.who", { version: 1, - input: Type.Object({}), - members: { who: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { who: method(undefined, str()) }, }); describe("principals", () => { @@ -86,9 +86,9 @@ describe("principals", () => { it("a child's parent acts as the principal the child gave `connect`; relayed callers keep theirs", async () => { const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("worker"), - variants: { main: { methods: { who: method(undefined, Type.String()) } } }, + variants: { main: { methods: { who: method(undefined, str()) } } }, }); const server = createHost({ name: "server", self: { kind: "host" }, principalKey }); const [down, up] = memoryLinkPair(); diff --git a/packages/chord/test/provider-cost.test.ts b/packages/chord/test/provider-cost.test.ts index 470c74b20..50a8bf533 100644 --- a/packages/chord/test/provider-cost.test.ts +++ b/packages/chord/test/provider-cost.test.ts @@ -1,7 +1,7 @@ // Entry 8: what a change costs the provider. The diff is computed once per change; each feed (one per connection and // member) encodes it with its own stateful encoder and its link serializes it. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { describe, it, vi } from "vitest"; import type * as delta from "../src/delta/index.ts"; import { type Ctx, createHost, hostNamed, memoryLinkPair, method, remote } from "../src/index.ts"; @@ -30,16 +30,16 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); const List = remote("t.e8-cost", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { items: Type.Array(Type.String()) }, methods: { add: method(Type.String()) } } }, + variants: { main: { state: { items: arr(str()) }, methods: { add: method(str()) } } }, }); describe("provider cost", () => { it("one change to K clients: K encodes (one per feed), one frame per link, the same ops on every link", async () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(List, (_input, _principal, self) => { - const items = self.state("items", Type.Array(Type.String()), []); + const items = self.state("items", arr(str()), []); return { variant: "main", items, diff --git a/packages/chord/test/readiness.test.ts b/packages/chord/test/readiness.test.ts index a781a6e34..b34b2540c 100644 --- a/packages/chord/test/readiness.test.ts +++ b/packages/chord/test/readiness.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { createHost, @@ -15,8 +15,8 @@ import { const tick = () => new Promise((resolve) => setImmediate(resolve)); const unexpectedError = (error: unknown) => assert.fail(String(error)); -const Eager = local("ready.eager", { version: 1, input: Type.Object({}) }); -const Lazy = local("ready.lazy", { version: 1, input: Type.Object({}) }); +const Eager = local("ready.eager", { version: 1, input: obj({}, { open: true }) }); +const Lazy = local("ready.lazy", { version: 1, input: obj({}, { open: true }) }); // Startup must not report ready while an eager object has failed. describe("host: eager readiness", () => { @@ -162,7 +162,7 @@ describe("host: eager readiness", () => { it("ready includes direct host serves, but not later serves", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const First = local("ready.first", { version: 1, input: Type.Object({}) }); + const First = local("ready.first", { version: 1, input: obj({}, { open: true }) }); let created = 0; try { host.serve( @@ -201,7 +201,7 @@ describe("host: eager readiness", () => { }); const Status = remote("ready.status", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { default: {} }, }); diff --git a/packages/chord/test/reload.test.ts b/packages/chord/test/reload.test.ts index f2931f0c9..b9998b316 100644 --- a/packages/chord/test/reload.test.ts +++ b/packages/chord/test/reload.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, opt, type Schema, str, unknown } from "@earendil-works/pi-schema"; import { describe, it, onTestFinished, vi } from "vitest"; import { type Ctx, @@ -30,15 +30,15 @@ function latch() { const Greeter = local("t.greeter", { version: 1, - input: Type.Object({ id: Type.Optional(Type.String({ default: "a" })) }), - members: { greet: method(Type.String(), Type.String()), count: Type.Unknown() }, + input: obj({ id: opt(str({ default: "a" })) }, { open: true }), + members: { greet: method(str(), str()), count: unknown() }, }); -const CountV1 = Type.Object({ n: Type.Number() }); +const CountV1 = obj({ n: num() }, { open: true }); interface GreeterSpec { readonly greeting: string; - readonly schema?: typeof CountV1 | ReturnType; + readonly schema?: typeof CountV1 | Schema; readonly initial?: object; readonly bump?: (draft: Record) => void; readonly options?: ServeOptions; @@ -129,7 +129,7 @@ describe("reload", () => { const writing = greeterFacet({ greeting: "hi", extra: (self) => { - const probe = self.state("probe", Type.Object({}), {}); + const probe = self.state("probe", obj({}, { open: true }), {}); probe.replace(captured!, {}); }, }); @@ -164,7 +164,7 @@ describe("reload", () => { await host.hold(Greeter, {}).ready; const changed = greeterFacet({ greeting: "hi", - schema: Type.Object({ total: Type.Number() }), + schema: obj({ total: num() }, { open: true }), initial: { total: 0 }, }); await assert.rejects( @@ -191,7 +191,7 @@ describe("reload", () => { const froms: number[] = []; const v2 = greeterFacet({ greeting: "hi", - schema: Type.Object({ total: Type.Number() }), + schema: obj({ total: num() }, { open: true }), initial: { total: 0 }, bump: (draft) => { draft.total = (draft.total ?? 0) + 1; @@ -218,16 +218,16 @@ describe("reload", () => { const host = quiet(); const Pair = local("t.pair", { version: 1, - input: Type.Object({}), - members: { a: Type.Unknown(), b: Type.Unknown(), bump: method() }, + input: obj({}, { open: true }), + members: { a: unknown(), b: unknown(), bump: method() }, }); const migrated: string[] = []; const facet = (version: number): Facet => ({ name: "pair", setup: (api) => api.serve(Pair, (_input, _principal, self) => { - const a = self.state("a", Type.Number(), 1); - const b = self.state("b", Type.Number(), 1, { + const a = self.state("a", num(), 1); + const b = self.state("b", num(), 1, { version, migrate: (old, from) => { migrated.push(`b ${from}->${version}`); @@ -294,7 +294,7 @@ describe("reload", () => { await new Promise((resolve) => ctx.signal.addEventListener("abort", () => resolve())); log.push(`stop ${greeting}`); }); - const count = self.state("count", Type.Number(), 0, { version }); + const count = self.state("count", num(), 0, { version }); return { count, greet: (name: string) => `${greeting} ${name}` }; }), }); @@ -340,7 +340,7 @@ describe("reload", () => { open: () => void log.push(`open cache ${rev}`), close: () => void log.push(`close cache ${rev}`), }); - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); return { count, greet: (name: string) => `${lock.value} ${name}` }; }), }); @@ -523,21 +523,21 @@ describe("reload", () => { it("checks contracts at reload: forgotten bumps, live consumers, accepts adapters", async () => { const DocV1 = remote("t.rdoc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { read: method(undefined, Type.Object({ text: Type.String() })) } } }, + variants: { main: { methods: { read: method(undefined, obj({ text: str() }, { open: true })) } } }, }); const DocBroken = remote("t.rdoc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { read: method(undefined, Type.Object({ text: Type.Number() })) } } }, + variants: { main: { methods: { read: method(undefined, obj({ text: num() }, { open: true })) } } }, }); const DocV2 = remote("t.rdoc", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { fetch: method(undefined, Type.Object({ body: Type.String() })) } } }, + variants: { main: { methods: { fetch: method(undefined, obj({ body: str() }, { open: true })) } } }, }); const host = quiet(); host.install({ @@ -590,14 +590,14 @@ describe("reload", () => { const log: string[] = []; const Sum = local("t.sum", { version: 1, - input: Type.Object({}), - members: { sum: Type.Object({ v: Type.Number() }) }, + input: obj({}, { open: true }), + members: { sum: obj({ v: num() }, { open: true }) }, }); const facet = (factor: number): Facet => ({ name: "sum", setup: (api) => api.serve(Sum, (_input, _principal, self) => { - const sum = self.state("sum", Type.Object({ v: Type.Number() }), { v: 0 }); + const sum = self.state("sum", obj({ v: num() }, { open: true }), { v: 0 }); self.on( "add", events, @@ -632,7 +632,7 @@ describe("reload", () => { it("keeps registry tables per code version", async () => { const host = quiet(); const Routes = defineRegistry("t.reload-routes"); - const Router = local("t.router", { version: 1, input: Type.Object({}) }); + const Router = local("t.router", { version: 1, input: obj({}, { open: true }) }); const facet = (value: string): Facet => ({ name: "router", setup: (api) => @@ -672,8 +672,8 @@ describe("reload", () => { await ref.ready; const Clock = local("t.clock", { version: 1, - input: Type.Object({}), - members: { now: method(undefined, Type.Number()) }, + input: obj({}, { open: true }), + members: { now: method(undefined, num()) }, }); const clockFacet: Facet = { name: "clock", setup: (api) => api.serve(Clock, () => ({ now: () => 42 })) }; const consumer = (fail: boolean): Facet => ({ diff --git a/packages/chord/test/requests.test.ts b/packages/chord/test/requests.test.ts index fef1f2a30..5e3cc5abb 100644 --- a/packages/chord/test/requests.test.ts +++ b/packages/chord/test/requests.test.ts @@ -1,6 +1,6 @@ // Regression tests for resolved application requests (#1-#10), numbered as in their test names. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { applyImmutable } from "../src/delta/index.ts"; import { @@ -36,19 +36,19 @@ function latch() { return { promise, open }; } -const View = Type.Object({ text: Type.String() }); +const View = obj({ text: str() }, { open: true }); describe("application requests", () => { it("#1 a held object's state member can be returned as this object's state member", async () => { const Conv = remote("t.conv", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { view: View }, methods: { set: method(Type.String()) } } }, + variants: { main: { state: { view: View }, methods: { set: method(str()) } } }, }); const Chat = remote("t.chat", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { view: View } } }, }); @@ -95,11 +95,11 @@ describe("application requests", () => { it("#2 facets reach their own host through FacetApi.root and FacetApi.registry", async () => { const Config = defineRegistry("t.config"); - const Settings = local("t.settings", { version: 1, input: Type.Object({}) }); + const Settings = local("t.settings", { version: 1, input: obj({}, { open: true }) }); const Reader = local("t.reader", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, str()) }, }); const captured: FacetApi[] = []; const facet = (value: string): Facet => ({ @@ -135,8 +135,8 @@ describe("application requests", () => { it("#3 a listener keeps its subscription across reloads when resource and select are unchanged", async () => { const Watcher = local("t.watcher", { version: 1, - input: Type.Object({}), - members: { seen: Type.Array(Type.String()) }, + input: obj({}, { open: true }), + members: { seen: arr(str()) }, }); // A source that accepts one subscriber at a time (like a Durable watch). let subscribers = 0; @@ -155,7 +155,7 @@ describe("application requests", () => { name: "watcher", setup: (api) => api.serve(Watcher, (_input, _principal, self) => { - const seen = self.state("seen", Type.Array(Type.String()), []); + const seen = self.state("seen", arr(str()), []); const watchResource = self.resource("watch", { version: 1, open: () => single }); // A new adapter object per call, and a new arrow per factory run. self.on( @@ -181,7 +181,7 @@ describe("application requests", () => { }); it("#3 a changed select resubscribes", async () => { - const Watcher = local("t.watcher2", { version: 1, input: Type.Object({}) }); + const Watcher = local("t.watcher2", { version: 1, input: obj({}, { open: true }) }); let subscribes = 0; const source = { subscribe: () => { @@ -210,10 +210,10 @@ describe("application requests", () => { it("#4 replace diffs a new revision that shares structure; unrelated values replace", async () => { const Mirror = local("t.mirror", { version: 1, - input: Type.Object({}), - members: { doc: Type.Unknown(), set: method(Type.Unknown()) }, + input: obj({}, { open: true }), + members: { doc: unknown(), set: method(unknown()) }, }); - const Doc = Type.Object({ entries: Type.Array(Type.String()), status: Type.String() }); + const Doc = obj({ entries: arr(str()), status: str() }, { open: true }); let lastOptions: { diff?: boolean } | undefined; const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Mirror, (_input, _principal, self) => { @@ -256,9 +256,9 @@ describe("application requests", () => { it("#5 roots of a child that connected behind the server are admitted in the server's code version", async () => { const Echo = remote("t.echo", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { codeVersion: method(undefined, Type.Number()) } } }, + variants: { main: { methods: { codeVersion: method(undefined, num()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); const echoFacet: Facet = { @@ -293,7 +293,7 @@ describe("application requests", () => { it("#6 self.expose: an opaque member backed by a resource, readable once live", async () => { const Store = local("t.store", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ readonly queries: string[] }>("t.db@1") }, }); const gate = latch(); @@ -315,7 +315,7 @@ describe("application requests", () => { await ref.ready; assert.equal(ref.db, native); - const Other = local("t.other", { version: 1, input: Type.Object({}) }); + const Other = local("t.other", { version: 1, input: obj({}, { open: true }) }); host.serve(Other, (_input, _principal, self) => { self.expose({ key: "x", value: 1 }); return {}; @@ -326,9 +326,9 @@ describe("application requests", () => { it("#7 a cluster reload reports what each child prepared", async () => { const Greeter = remote("t.greeter", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("session:S1"), - variants: { main: { methods: { greet: method(Type.String(), Type.String()) } } }, + variants: { main: { methods: { greet: method(str(), str()) } } }, }); const greeter = (text: string): Facet => ({ name: "greeter", @@ -362,13 +362,13 @@ describe("application requests", () => { it("#8 a held remote reference keeps its state without listeners; a later subscriber sees the current value", async () => { const Counter = remote("t.count8", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() }, methods: { set: method(Type.Number()) } } }, + variants: { main: { state: { n: num() }, methods: { set: method(num()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Counter, (_input, _principal, self) => { - const n = self.state("n", Type.Number(), 0); + const n = self.state("n", num(), 0); return { variant: "main", n, @@ -404,7 +404,7 @@ describe("application requests", () => { it("#9 references report status changes: live, loading while a link reconnects, gone, released", async () => { const Thing = remote("t.thing9", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: {} }, }); @@ -449,9 +449,9 @@ describe("application requests", () => { const log: string[] = []; const Remote = remote("t.remote10", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { methods: { note: method(Type.String()) } } }, + variants: { main: { methods: { note: method(str()) } } }, }); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Remote, () => ({ @@ -462,10 +462,10 @@ describe("application requests", () => { })); const Proc = local("t.proc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { tracker: opaque>("t.tracker@1") }, }); - const Chat = local("t.chat10", { version: 1, input: Type.Object({}) }); + const Chat = local("t.chat10", { version: 1, input: obj({}, { open: true }) }); const [down, up] = memoryLinkPair(); server.accept(down, { principal: { worker: "w" }, kind: "host", name: "w" }); const worker = createHost({ diff --git a/packages/chord/test/review.test.ts b/packages/chord/test/review.test.ts index 554a24097..418d592ea 100644 --- a/packages/chord/test/review.test.ts +++ b/packages/chord/test/review.test.ts @@ -1,7 +1,6 @@ // Regression tests for Sol's phase 1a review (findings #1-#11) and the requested race tests. import assert from "node:assert/strict"; -import { Type } from "typebox"; -import { Value } from "typebox/value"; +import { arr, bool, compile, num, obj, opt, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -18,6 +17,7 @@ import { } from "../src/index.ts"; import { fits } from "../src/schema.ts"; import { CountingLink } from "./fixtures/counting.ts"; +import { typedLit } from "./fixtures/schemas.ts"; const tick = () => new Promise((resolve) => setImmediate(resolve)); const ticks = async (n = 10) => { @@ -51,8 +51,8 @@ function emitter(options: { failSubscribe?: boolean } = {}) { const Item = local("t.item", { version: 1, - input: Type.Object({ id: Type.String() }), - members: { version: method(undefined, Type.String()) }, + input: obj({ id: str() }, { open: true }), + members: { version: method(undefined, str()) }, }); describe("review: reload commit", () => { @@ -82,15 +82,15 @@ describe("review: reload commit", () => { const host = createHost({ self: { kind: "host" }, dev: true, onError: () => {} }); const V1 = remote("t.counted", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { state: { count: Type.Object({ n: Type.Number() }) } } }, + variants: { main: { state: { count: obj({ n: num() }, { open: true }) } } }, }); const V2 = remote("t.counted", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { state: { count: Type.Object({ total: Type.Number() }) } } }, + variants: { main: { state: { count: obj({ total: num() }, { open: true }) } } }, }); host.install({ name: "counted", @@ -99,7 +99,7 @@ describe("review: reload commit", () => { V1, (_input, _principal, self) => ({ variant: "main", - count: self.state("count", Type.Object({ n: Type.Number() }), { n: 3 }), + count: self.state("count", obj({ n: num() }, { open: true }), { n: 3 }), }), { linger: 60_000 }, ), @@ -117,7 +117,7 @@ describe("review: reload commit", () => { variant: "main", count: self.state( "count", - Type.Object({ total: Type.Number() }), + obj({ total: num() }, { open: true }), { total: 0 }, { version: 2, migrate: (old) => ({ total: (old as { n: number }).n }) }, ), @@ -160,8 +160,8 @@ describe("review: reload commit", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Other = local("t.other", { version: 1, - input: Type.Object({}), - members: { version: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { version: method(undefined, str()) }, }); const facet = (version: string): Facet => ({ name: "pair", @@ -184,8 +184,8 @@ describe("review: reload commit", () => { describe("review: leases, invocations, activation", () => { const Store = local("t.store", { version: 1, - input: Type.Object({}), - members: { count: Type.Unknown() }, + input: obj({}, { open: true }), + members: { count: unknown() }, }); it("#4 a running listener holds a lease: the gate waits for it before migrating", async () => { @@ -197,7 +197,7 @@ describe("review: leases, invocations, activation", () => { name: "store", setup: (api) => api.serve(Store, (_input, _principal, self) => { - const count = self.state("count", Type.Object({ n: Type.Number() }), { n: 1 }); + const count = self.state("count", obj({ n: num() }, { open: true }), { n: 1 }); self.on("slow", events, async () => { await go.promise; seen.push(count.value); @@ -218,7 +218,7 @@ describe("review: leases, invocations, activation", () => { api.serve(Store, (_input, _principal, self) => ({ count: self.state( "count", - Type.Object({ total: Type.Number() }), + obj({ total: num() }, { open: true }), { total: 0 }, { version: 2, migrate: (old) => ({ total: (old as { n: number }).n }) }, ), @@ -238,8 +238,8 @@ describe("review: leases, invocations, activation", () => { it("#5 disposal waits for running methods before closing resources (released and forced)", async () => { const Worker = local("t.busy", { version: 1, - input: Type.Object({ id: Type.String() }), - members: { work: method(undefined, Type.Boolean()) }, + input: obj({ id: str() }, { open: true }), + members: { work: method(undefined, bool()) }, }); const log: string[] = []; let go = latch(); @@ -291,7 +291,9 @@ describe("review: leases, invocations, activation", () => { const host = createHost({ self: { kind: "host" }, onError: (error) => errors.push(error) }); const Routes = defineRegistry("t.review-routes"); const log: string[] = []; - const types = ["t.eager-a", "t.eager-b"].map((name) => local(name, { version: 1, input: Type.Object({}) })); + const types = ["t.eager-a", "t.eager-b"].map((name) => + local(name, { version: 1, input: obj({}, { open: true }) }), + ); for (const type of types) { host.serve( type, @@ -321,7 +323,7 @@ describe("review: leases, invocations, activation", () => { let runs = 0; host.serve(Store, (_input, _principal, self) => { runs++; - const count = self.state("count", Type.Number(), 0); + const count = self.state("count", num(), 0); self.resource("r", { version: 1, open: () => void log.push("open"), @@ -345,8 +347,8 @@ describe("review: leases, invocations, activation", () => { describe("review: prepare isolation and contracts", () => { const Db = local("t.db", { version: 1, - input: Type.Object({}), - members: { read: method(undefined, Type.Unknown()) }, + input: obj({}, { open: true }), + members: { read: method(undefined, unknown()) }, }); it("#7 a prepare rerun cannot touch retained resources, even open ones", async () => { @@ -372,7 +374,7 @@ describe("review: prepare isolation and contracts", () => { it("#7 objects of types a reload adds start listeners and tasks only at commit", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const log: string[] = []; - const Fresh = local("t.fresh", { version: 1, input: Type.Object({}) }); + const Fresh = local("t.fresh", { version: 1, input: obj({}, { open: true }) }); host.install({ name: "item", setup: (api) => api.serve(Item, () => ({ version: () => "v1" })) }); await Promise.all(["a", "b"].map((id) => host.hold(Item, { id }).ready)); const plugins = (fail: boolean): Facet[] => [ @@ -408,20 +410,20 @@ describe("review: prepare isolation and contracts", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const P1 = remote("t.provider", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { get: method(undefined, Type.String()) } } }, + variants: { main: { methods: { get: method(undefined, str()) } } }, }); const P2 = remote("t.provider", { version: 2, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), - variants: { main: { methods: { fetch: method(undefined, Type.String()) } } }, + variants: { main: { methods: { fetch: method(undefined, str()) } } }, }); const Consumer = local("t.consumer", { version: 1, - input: Type.Object({}), - members: { ask: method(undefined, Type.String()) }, + input: obj({}, { open: true }), + members: { ask: method(undefined, str()) }, }); host.install({ name: "both", @@ -493,7 +495,7 @@ describe("review: prepare isolation and contracts", () => { it("#10 retirement closes dropped resources before releasing dropped holds", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const log: string[] = []; - const Dep = local("t.dep", { version: 1, input: Type.Object({}) }); + const Dep = local("t.dep", { version: 1, input: obj({}, { open: true }) }); const facet = (withDecls: boolean): Facet => ({ name: "item", setup: (api) => { @@ -535,48 +537,49 @@ describe("review: prepare isolation and contracts", () => { assert.deepEqual(log, ["own closed", "dep closed"]); }); - it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with Value.Check", () => { + it("#11 fits has no false positives for unmodelled keywords; counterexamples checked with compile", () => { const cases = [ - [Type.String(), Type.String({ not: { const: "x" } }), "x"], - [Type.Array(Type.Number()), Type.Array(Type.Number(), { contains: Type.Literal(1) }), [2]], + [str(), { ...str(), not: { const: "x" } }, "x"], + [arr(num()), { ...arr(num()), contains: typedLit(1) }, [2]], [ - Type.Object({ a: Type.String() }), - Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), + obj({ a: str() }, { open: true }), + { ...obj({ a: str() }, { open: true }), additionalProperties: num() }, { a: "s", b: "t" }, ], [ - Type.Object({ x: Type.String() }), - Type.Object({}, { patternProperties: { "^x$": Type.Number() } }), + obj({ x: str() }, { open: true }), + { ...obj({}, { open: true }), patternProperties: { "^x$": num() } }, { x: "s" }, ], - [Type.Number(), Type.Intersect([Type.Number(), Type.Number({ minimum: 1 })]), 0], + [num(), { allOf: [num(), num({ minimum: 1 })] }, 0], [ - Type.Object({ a: Type.String() }, { additionalProperties: Type.Number() }), - Type.Object({ a: Type.String(), b: Type.Optional(Type.String()) }), + { ...obj({ a: str() }, { open: true }), additionalProperties: num() }, + obj({ a: str(), b: opt(str()) }, { open: true }), { a: "", b: 1 }, ], ] as const; for (const [sub, sup, value] of cases) { - assert.ok(Value.Check(sub, value) && !Value.Check(sup, value), `bad counterexample ${JSON.stringify(value)}`); + assert.ok( + compile(sub).check(value) && !compile(sup).check(value), + `bad counterexample ${JSON.stringify(value)}`, + ); assert.equal(fits(sub, sup), false, `fits(${JSON.stringify(sub)}, ${JSON.stringify(sup)})`); } // Additive changes still fit. - assert.ok( - fits(Type.Object({ a: Type.String() }), Type.Object({ a: Type.String(), b: Type.Optional(Type.Number()) })), - ); - assert.ok(fits(Type.Object({ a: Type.String(), extra: Type.Number() }), Type.Object({ a: Type.String() }))); + assert.ok(fits(obj({ a: str() }, { open: true }), obj({ a: str(), b: opt(num()) }, { open: true }))); + assert.ok(fits(obj({ a: str(), extra: num() }, { open: true }), obj({ a: str() }, { open: true }))); }); it("#11 arguments are stripped to what the consumer knows, then to what the provider knows", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Strict = remote("t.strict", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("host"), variants: { main: { methods: { - echo: method(Type.Object({ a: Type.String() }, { additionalProperties: false }), Type.Unknown()), + echo: method(obj({ a: str() }), unknown()), }, }, }, @@ -590,13 +593,13 @@ describe("review: prepare isolation and contracts", () => { describe("review: ctx races", () => { const Target = local("t.target", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { hit: method() }, }); it("an aborted task ctx cannot start fresh calls", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Runner = local("t.runner", { version: 1, input: Type.Object({}) }); + const Runner = local("t.runner", { version: 1, input: obj({}, { open: true }) }); let hits = 0; const go = latch(); let outcome: unknown; @@ -650,7 +653,7 @@ describe("review: ctx races", () => { it("a remote reference refuses a foreign ctx and a ctx not made by Chord, without calling", async () => { const RemoteTarget = remote("t.remote-target", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { state: {}, methods: { hit: method() } } }, }); diff --git a/packages/chord/test/state-init.test.ts b/packages/chord/test/state-init.test.ts index 77f101d5e..24d086aac 100644 --- a/packages/chord/test/state-init.test.ts +++ b/packages/chord/test/state-init.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, bool, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, createHost, type Facet, isChordError, local, method } from "../src/index.ts"; @@ -19,16 +19,16 @@ function latch() { const Notes = local("t.notes", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { - notes: Type.Array(Type.String()), - count: Type.Number(), - has: method(Type.String(), Type.Boolean()), - add: method(Type.String()), + notes: arr(str()), + count: num(), + has: method(str(), bool()), + add: method(str()), }, }); -const NotesSchema = Type.Array(Type.String()); +const NotesSchema = arr(str()); describe("state init", () => { it("runs after the resources are open and before the object goes live: nothing sees another value", async () => { @@ -124,7 +124,7 @@ describe("state init", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Source = local("t.source", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { all: method(undefined, NotesSchema) }, }); host.serve(Source, () => ({ all: () => ["from source"] })); @@ -155,7 +155,7 @@ describe("state init", () => { return ["loaded"]; }, }); - const counter = self.state("counter", Type.Number(), 0, { version: spec.countVersion ?? 1 }); + const counter = self.state("counter", num(), 0, { version: spec.countVersion ?? 1 }); const extra = spec.extra; const more = extra === undefined diff --git a/packages/chord/test/state.test.ts b/packages/chord/test/state.test.ts index 656ac0601..2f48c8799 100644 --- a/packages/chord/test/state.test.ts +++ b/packages/chord/test/state.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { arr, num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { applyImmutable } from "../src/delta/index.ts"; import { type Ctx, createHost, isChordError, local, method, type StateEvent } from "../src/index.ts"; @@ -22,15 +22,15 @@ function emitter() { }; } -const Item = Type.Object({ id: Type.Number(), label: Type.String() }); -const Items = Type.Object({ items: Type.Array(Item), count: Type.Number() }); +const Item = obj({ id: num(), label: str() }, { open: true }); +const Items = obj({ items: arr(Item), count: num() }, { open: true }); const List = local("t.list", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { list: Items, - labels: Type.Array(Type.String()), + labels: arr(str()), push: method(Item), set: method(Items), }, @@ -84,11 +84,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Counter = local("t.prim", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), inc: method(undefined, Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), inc: method(undefined, num()) }, }); host.serve(Counter, (_input, _principal, self) => { - const nState = self.state("n", Type.Number(), 0); + const nState = self.state("n", num(), 0); return { n: nState, inc: (ctx: Ctx) => { @@ -117,11 +117,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Box = local("t.box", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { put: method() }, }); host.serve(Box, (_input, _principal, self) => { - const list = self.state("list", Type.Array(Type.Number()), []); + const list = self.state("list", arr(num()), []); return { put: (ctx: Ctx) => { // The classic slip: an expression-bodied arrow returns push()'s length. @@ -133,10 +133,10 @@ describe("state cells", () => { }); it("validates initial values always and writes only in dev mode", async () => { - const Bad = local("t.bad-initial", { version: 1, input: Type.Object({}) }); + const Bad = local("t.bad-initial", { version: 1, input: obj({}, { open: true }) }); const host = createHost({ self: { kind: "host" }, onError: () => {} }); host.serve(Bad, (_input, _principal, self) => { - self.state("x", Type.Object({ n: Type.Number() }), { n: "no" } as never); + self.state("x", obj({ n: num() }, { open: true }), { n: "no" } as never); return {}; }); await assert.rejects(host.hold(Bad, {}).ready, /initial value of state x/); @@ -157,13 +157,13 @@ describe("state cells", () => { it("rejects writes from inside a factory and placeholder use before activation", async () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); - const Writer = local("t.writer", { version: 1, input: Type.Object({}) }); + const Writer = local("t.writer", { version: 1, input: obj({}, { open: true }) }); let captured: Ctx | undefined; await host.root((ctx) => { captured = ctx; }); host.serve(Writer, (_input, _principal, self) => { - const x = self.state("x", Type.Object({}), {}); + const x = self.state("x", obj({}, { open: true }), {}); const r = self.resource("r", { version: 1, open: () => 1 }); assert.throws( () => r.value, @@ -180,13 +180,13 @@ describe("state cells", () => { const events = emitter(); const Watcher = local("t.watcher", { version: 1, - input: Type.Object({}), - members: { seen: Type.Object({ sum: Type.Number() }) }, + input: obj({}, { open: true }), + members: { seen: obj({ sum: num() }, { open: true }) }, }); const roots = new Set(); let taskAborted = false; host.serve(Watcher, (_input, _principal, self) => { - const seen = self.state("seen", Type.Object({ sum: Type.Number() }), { sum: 0 }); + const seen = self.state("seen", obj({ sum: num() }, { open: true }), { sum: 0 }); const sourceResource = self.resource("source", { version: 1, open: () => events }); self.on( "onEvent", @@ -223,11 +223,11 @@ describe("state cells", () => { const host = createHost({ self: { kind: "host" }, onError: () => {} }); const Echo = local("t.reentrant", { version: 1, - input: Type.Object({}), - members: { n: Type.Number(), d: Type.Number(), set: method(Type.Number()) }, + input: obj({}, { open: true }), + members: { n: num(), d: num(), set: method(num()) }, }); host.serve(Echo, (_input, _principal, self) => { - const nState = self.state("n", Type.Number(), 0); + const nState = self.state("n", num(), 0); const d = self.derive([nState], (n) => n * 2); // Subscribed first: re-enters the write while revision 1 is being published. self.on("bump", nState, (n, ctx) => { diff --git a/packages/chord/test/tutorial.test.ts b/packages/chord/test/tutorial.test.ts index 55c0dfe17..72f98e1dd 100644 --- a/packages/chord/test/tutorial.test.ts +++ b/packages/chord/test/tutorial.test.ts @@ -1,7 +1,7 @@ // example:imports import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; -import { type Static, Type } from "typebox"; +import { arr, bool, int, num, obj, type Static, str } from "@earendil-works/pi-schema"; import { it, onTestFinished } from "vitest"; import { type Ctx, @@ -51,23 +51,23 @@ function checkExamples(section: string): void { // example:02-contract const TodoLabel = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { label: method(undefined, Type.String()) }, + input: obj({ list: str() }, { open: true }), + members: { label: method(undefined, str()) }, }); // end-example:02-contract // example:03-contract -const Todo = Type.Object({ text: Type.String(), done: Type.Boolean() }); -const Rows = Type.Array(Todo); +const Todo = obj({ text: str(), done: bool() }, { open: true }); +const Rows = arr(Todo); const TodoStore = local("tutorial.todos", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), members: { - label: method(undefined, Type.String()), + label: method(undefined, str()), rows: Rows, - openCount: Type.Number(), - add: method(Type.Object({ text: Type.String() })), - finish: method(Type.Object({ index: Type.Integer() })), + openCount: num(), + add: method(obj({ text: str() }, { open: true })), + finish: method(obj({ index: int() }, { open: true })), reset: method(Rows), }, }); @@ -107,13 +107,13 @@ const todoFactory = todoStore(); // example:05-views const Todos = remote("tutorial.todo-view", { version: 1, - input: Type.Object({ list: Type.String() }), + input: obj({ list: str() }, { open: true }), host: hostNamed("server"), variants: { viewer: { state: { rows: Rows } }, editor: { state: { rows: Rows }, - methods: { add: method(Type.Object({ text: Type.String() })) }, + methods: { add: method(obj({ text: str() }, { open: true })) }, }, }, }); @@ -190,13 +190,13 @@ it("04: holds, asynchronous resources and release", { timeout: 5000 }, async () // example:04-database const Database = local("tutorial.database", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), members: { db: opaque<{ saved: string[] }>("tutorial.database@1") }, }); const SavedTodos = local("tutorial.saved-todos", { version: 1, - input: Type.Object({ list: Type.String() }), - members: { add: method(Type.Object({ text: Type.String() })) }, + input: obj({ list: str() }, { open: true }), + members: { add: method(obj({ text: str() }, { open: true })) }, }); const host = createHost({ self: "server" }); let opens = 0; @@ -306,8 +306,8 @@ it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async ( // example:07 const Activity = local("tutorial.activity", { version: 1, - input: Type.Object({}), - members: { seen: Type.Number() }, + input: obj({}, { open: true }), + members: { seen: num() }, }); const host = createHost({ self: "server" }); host.serve(TodoStore, todoFactory); @@ -316,7 +316,7 @@ it("07: listeners react and tasks start fresh calls", { timeout: 5000 }, async ( const stopped = Promise.withResolvers(); host.serve(Activity, (_input, _principal, self) => { const todos = self.get(TodoStore, { list: "family" }); - const seen = self.state("seen", Type.Number(), 0); + const seen = self.state("seen", num(), 0); self.on("rows", todos.rows, (rows, ctx) => { seen.replace(ctx, rows?.length ?? 0); if (rows?.length === 1) observed.resolve(); @@ -345,7 +345,7 @@ it("08: plugins contribute registry entries and expose callbacks", { timeout: 50 onTestFinished(() => host.close()); // example:08 const Actions = defineRegistry<(ctx: Ctx) => Promise>("tutorial.actions"); - const Menu = local("tutorial.menu", { version: 1, input: Type.Object({}) }); + const Menu = local("tutorial.menu", { version: 1, input: obj({}, { open: true }) }); let click: (() => Promise) | undefined; const storeFacet: Facet = { name: "todo-store", @@ -444,7 +444,7 @@ it("09: compatible reload, migration and an old-contract adapter", { timeout: 50 const RenamedStore = local("tutorial.todos", { version: 2, input: TodoStore.def.input, - members: { ...members, heading: method(undefined, Type.String()) }, + members: { ...members, heading: method(undefined, str()) }, }); await host.reload([ { diff --git a/packages/chord/test/types.test.ts b/packages/chord/test/types.test.ts index f64f0ab98..9df86f431 100644 --- a/packages/chord/test/types.test.ts +++ b/packages/chord/test/types.test.ts @@ -1,25 +1,25 @@ // Entry 8: reference types. State members are plain values (a reference you hold has its state); a remote reference is a // union discriminated by `variant`, so another variant's members are reachable only after narrowing. Checked by tsc. -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, expectTypeOf, it } from "vitest"; import { type Ctx, type Host, hostNamed, local, method, type Readable, type Ref, remote } from "../src/index.ts"; const Doc = remote("t.e8-types-doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - reader: { state: { text: Type.String() } }, - editor: { state: { text: Type.String() }, methods: { edit: method(Type.String()) } }, + reader: { state: { text: str() } }, + editor: { state: { text: str() }, methods: { edit: method(str()) } }, }, }); const Single = remote("t.e8-types-single", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), - variants: { main: { state: { n: Type.Number() }, methods: { bump: method() } } }, + variants: { main: { state: { n: num() }, methods: { bump: method() } } }, }); -const Local = local("t.e8-types-local", { version: 1, input: Type.Object({}), members: { n: Type.Number() } }); +const Local = local("t.e8-types-local", { version: 1, input: obj({}, { open: true }), members: { n: num() } }); describe("reference types", () => { it("state members are T; other variants' members need narrowing; gets are awaited", () => { diff --git a/packages/chord/test/variants.test.ts b/packages/chord/test/variants.test.ts index 363f50361..3b8c2359d 100644 --- a/packages/chord/test/variants.test.ts +++ b/packages/chord/test/variants.test.ts @@ -1,7 +1,7 @@ // Entry 8: a binding applies a shape only once its new state members have snapshots; members outside the variant // throw `not_in_variant`; a re-hold after a link loss keeps the last values readable and sends one fresh unit. import assert from "node:assert/strict"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { type Ctx, @@ -24,12 +24,12 @@ async function until(check: () => boolean, attempts = 200): Promise { const Doc = remote("t.e8-doc", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { - a: { state: { x: Type.Number() }, methods: { bump: method() } }, - b: { state: { x: Type.Number(), y: Type.String() }, methods: { bump: method() } }, - c: { state: { y: Type.String() }, methods: { bump: method() } }, + a: { state: { x: num() }, methods: { bump: method() } }, + b: { state: { x: num(), y: str() }, methods: { bump: method() } }, + c: { state: { y: str() }, methods: { bump: method() } }, }, }); @@ -39,8 +39,8 @@ function docFacet(variant: "a" | "b" | "c"): Facet { name: "doc", setup: (api) => api.serve(Doc, (_input, _principal, self) => { - const x = self.state("x", Type.Number(), 1); - const y = self.state("y", Type.String(), "why"); + const x = self.state("x", num(), 1); + const y = self.state("y", str(), "why"); const bump = (ctx: Ctx) => { x.replace(ctx, x.value + 1); }; diff --git a/packages/chord/test/websocket.test.ts b/packages/chord/test/websocket.test.ts index 8cb1e95c2..b5bc4389b 100644 --- a/packages/chord/test/websocket.test.ts +++ b/packages/chord/test/websocket.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import type { IncomingMessage } from "node:http"; -import { Type } from "typebox"; +import { arr, obj, str, unknown } from "@earendil-works/pi-schema"; import { describe, it } from "vitest"; import { WebSocketServer } from "ws"; import { @@ -17,12 +17,12 @@ import { const Board = remote("t.board", { version: 1, - input: Type.Object({}), + input: obj({}, { open: true }), host: hostNamed("server"), variants: { main: { - state: { notes: Type.Array(Type.String()) }, - methods: { add: method(Type.String()), who: method(undefined, Type.Unknown()) }, + state: { notes: arr(str()) }, + methods: { add: method(str()), who: method(undefined, unknown()) }, }, }, }); @@ -31,7 +31,7 @@ describe("WebSocket links", () => { it("serves a browser-style client over a real socket, stamped by the authenticator", async () => { const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Board, (_input, principal, self) => { - const notes = self.state("notes", Type.Array(Type.String()), []); + const notes = self.state("notes", arr(str()), []); return { variant: "main", notes, @@ -112,7 +112,7 @@ describe("WebSocket links: packed payloads", () => { const notes = Array.from({ length: 40 }, (_, i) => `${i}:${"x".repeat(4000)}`); const server = createHost({ self: { kind: "host" }, name: "server", onError: () => {} }); server.serve(Board, (_input, _principal, self) => { - const state = self.state("notes", Type.Array(Type.String()), notes); + const state = self.state("notes", arr(str()), notes); return { variant: "main", notes: state, add: () => {}, who: () => notes }; }); const wss = new WebSocketServer({ port: 0, host: "127.0.0.1" }); diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index decd59f9e..6c6de2736 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Changed + +- README: the codemode tool example builds its parameters with `@earendil-works/pi-schema` instead of TypeBox. + ## [1.4.0] - 2026-10-21 ## [1.3.0] - 2026-10-15 diff --git a/packages/codemode/README.md b/packages/codemode/README.md index e4fa87786..d97bd6bf5 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -275,7 +275,7 @@ import { type CodemodeTool, renderDeclarations, } from "@earendil-works/pi-codemode"; -import { Type } from "typebox"; +import { obj, str } from "@earendil-works/pi-schema"; const sandboxTools: CodemodeTool[] = agentTools.map((tool) => ({ name: tool.name, @@ -293,7 +293,7 @@ const codemodeTool: AgentTool = { name: "codemode", label: "Codemode", description: `Run JavaScript that calls tools as \`await tools.(args)\`. Output with text() or return.\n\n${renderDeclarations({ tools: sandboxTools })}`, - parameters: Type.Object({ code: Type.String() }), + parameters: obj({ code: str() }), execute: async (_toolCallId, { code }, signal) => { const sandbox = new CodemodeSandbox({ tools: sandboxTools }); try { diff --git a/packages/codemode/package.json b/packages/codemode/package.json index b1f0fea9c..006549146 100644 --- a/packages/codemode/package.json +++ b/packages/codemode/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-codemode", - "version": "1.4.0", + "version": "1.5.0", "description": "Sandboxed JavaScript execution where the only capability is calling injected tools", "type": "module", "main": "./dist/index.js", @@ -42,7 +42,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -70,7 +70,6 @@ "quickjs-wasi": "3.6.2" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 31571b8e9..c987db53b 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -4,17 +4,54 @@ ### Added +- `pi update` now lists each updated package with its npm version or git commit change, for example `npm:foo 1.2.0 -> 1.3.0`, and prints `Packages are already up to date` when nothing changed. When nothing changed but an update printed a warning, for example because npm left the old version installed after resolving stale registry metadata, it prints `No packages were updated, see the warnings above` instead; in offline mode it prints `Skipped package updates (offline mode)`. `PackageManager.update()` reports each changed package as a new `result` progress event whose `message` holds that line ([#11707](https://github.com/earendil-works/pi/pull/11707), [#11711](https://github.com/earendil-works/pi/pull/11711)) + +## [1.5.0] - 2026-11-02 + +### New Features + +- **Extension update changelogs**: see what changed in installed extension packages on the next fresh launch after an update. Package authors can choose the changelog path, and `collapseChangelog` condenses the notices. See [Show a changelog](docs/packages.md#show-a-changelog). +- **Faster streaming**: large tool calls are parsed incrementally, and code-heavy answers and tool rows reuse highlighting and rendered Markdown instead of repeating the work on every update. See [Keep rendering responsive](docs/tui.md#keep-rendering-responsive). +- **Custom self-update commands**: use `PI_SELF_UPDATE_COMMAND` or `selfUpdateCommand` to update pi through a wrapper or another package manager. See [Custom update command](docs/cli.md#custom-update-command). +- **GPT-6.1 Terra**: available through OpenAI, Azure OpenAI Responses, and OpenAI Codex, with tool search, mid-conversation thinking-level changes, and thinking `off`. Codex offers a 500K context window. See [Models](docs/cli.md#models). + +### Breaking Changes + +- Tool argument validation runs on `@earendil-works/pi-schema`. TypeBox features outside JSON Schema (`Type.Refine()`, `Type.BigInt()`, `Type.Undefined()`, codecs, `Format.Set()` formats) no longer apply to tool arguments; see [Tool parameter schemas and TypeBox](docs/extensions.md#tool-parameter-schemas-and-typebox). Plain JSON Schema tools, including MCP tools, now get the same coercion as TypeBox tools, such as converting numeric strings and wrapping a single value into an array. Validation failures the model sees list pi-schema's messages (`- path: missing`, `- limit: expected number, got string`) instead of TypeBox's (`must have required properties path`, `must be number`). + +### Added + - Extension packages can now show their changelog after an update. Pi reads `pi.changelogPath` from the package's `package.json`, or `CHANGELOG.md` in the package root, and on the next fresh launch shows entries newer than the last seen version under "Extension Updates". The `collapseChangelog` setting condenses this to one line per package ([#5958](https://github.com/earendil-works/pi/issues/5958), [#8790](https://github.com/earendil-works/pi/pull/8790)) +- `pi.exec()` accepts a `maxBuffer` option that limits stdout and stderr combined, and its result has a `truncated` field that is true when output exceeded the limit ([#11360](https://github.com/earendil-works/pi/issues/11360)) - Device-code login (for example GitHub Copilot) now opens the verification page and copies the code when you press Enter. Providers can turn either action off with `openBrowserOnConfirm: false` or `copyCodeOnConfirm: false` ([#9282](https://github.com/earendil-works/pi/issues/9282), [#9301](https://github.com/earendil-works/pi/pull/9301)) +- `pi update` can update Pi through a wrapper or another package manager. Set `PI_SELF_UPDATE_COMMAND` (for example `mise use -g npm:{packageName}@{version}`) or the global `selfUpdateCommand` setting; `pi update` runs it when pi.dev reports a newer release and fills `{packageName}`, `{version}`, and `{installSpec}` from that release. See [Custom update command](docs/cli.md#custom-update-command) ([#7756](https://github.com/earendil-works/pi/issues/7756), [#5607](https://github.com/earendil-works/pi/issues/5607)) +- Extensions can format the user's configured keybindings with `keyText()` and `keyHint()` from the tool render context and the message and entry renderer options. Importing these functions from `@earendil-works/pi-coding-agent` can resolve the extension's own copy of the package, which does not see pi's keybindings, so hints such as `keyHint("app.tools.expand", "to expand")` rendered without the key. The docs now recommend the injected functions ([#4748](https://github.com/earendil-works/pi/issues/4748)) +- Added persisted session-entry `messageId` and ISO `timestamp` to `MarkdownTransformContext`. They are available when rendering saved entries and undefined for live rendering ([#7952](https://github.com/earendil-works/pi/pull/7952) by [@vipentti](https://github.com/vipentti)) +- Added a `blocked` option to inline `ctx.ui.custom()` components so approval prompts and other custom dialogs can report that pi is waiting for user input. Overlays ignore the option. See [Custom screens and overlays](docs/tui.md#use-custom-screens-and-overlays) ([#11428](https://github.com/earendil-works/pi/issues/11428)) +- Added `Usage.cost.totalFromProvider` to distinguish billed totals from catalog estimates in SDK and extension usage data; combined usage is marked provider-reported only when every part is ([#11178](https://github.com/earendil-works/pi/pull/11178)) +- Added concrete `responseModel` metadata to messages from pi-messages providers such as Radius, including when a virtual model is requested. See [AssistantMessage](docs/message-types.md#assistantmessage) ([#11265](https://github.com/earendil-works/pi/pull/11265)) +- Added GPT-6.1 Terra for OpenAI, Azure OpenAI Responses, and OpenAI Codex, with tool search and mid-conversation thinking-level changes. It supports thinking `off`; Codex offers a 500K context window ### Changed +- Extensions keep importing `Type`, `Static` and `TSchema` from `@earendil-works/pi-ai`: Pi maps that import to pi-ai's compat entry plus the TypeBox it bundles, now that pi-ai itself no longer re-exports TypeBox, and its `StringEnum` there stays a TypeBox `TUnsafe`. The built-in tool factories (`createReadTool()` and the others) return `TypeBoxAgentTool`, which keeps `params` typed for TypeBox. +- Tightened system prompt - `pi update` for installs from the pi.dev installer no longer passes `--min-release-age=0` to `npm ci`. The release lockfile pins every package, so the flag had no effect, and security scanners that block it stopped the update ([#11262](https://github.com/earendil-works/pi/issues/11262)) - `/model` and `/scoped-models` now show a model's name next to its ID when the name adds words the ID lacks, such as a renamed `models.json` override ([#5916](https://github.com/earendil-works/pi/issues/5916), [#11347](https://github.com/earendil-works/pi/pull/11347)) +- MCP OAuth sign-in now uses pi's Client ID Metadata Document by default when the authorization server supports it for public clients, and registers a client dynamically otherwise, like Codex, Claude Code, and opencode. Clients registered before stay in use while their redirect URI matches. The new default `oauth.clientRegistration: "auto"` registers when `clientName` or a non-default `callbackUrl` is set; `"dcr"` always registers. `pi mcp add --oauth-client-registration` sets the mode ([#11426](https://github.com/earendil-works/pi/issues/11426)) +- Changed the default OpenAI model from `gpt-5.5` to `gpt-6.1-sol`, including after ChatGPT sign-in when no model is selected ([#11442](https://github.com/earendil-works/pi/issues/11442)) +- Updated the built-in model catalog to models.dev v2. Realtime and image-only models are no longer offered as chat models; newer OpenAI and Google chat models are included ### Fixed +- Updated glob matching to prevent denial of service from amplified glob patterns. +- Fixed a saved "Do not trust" decision being ignored for projects without trust-requiring resources, for example a folder with only `AGENTS.md`. pi auto-trusted such projects, so `ctx.isProjectTrusted()` returned true and package commands treated them as trusted. A saved decision in `trust.json` now takes precedence; auto-trust applies only when no decision is saved ([#10947](https://github.com/earendil-works/pi/issues/10947)) +- Fixed `Shift+Enter` queueing a follow-up instead of inserting a newline in Apple Terminal setups that send `ESC` + Return. pi now checks the local Shift key state for that sequence, as it already did for plain Return; `Alt+Enter` without Shift still queues a follow-up ([#11559](https://github.com/earendil-works/pi/issues/11559)) +- Fixed the working status (OSC 7501) disappearing in Rex and iTerm2 during runs in regular TUI mode. Rendered messages wrote OSC 133 prompt markers to the terminal, which the terminal treats as a new shell prompt; regular mode no longer writes them ([#11519](https://github.com/earendil-works/pi/issues/11519)) +- Fixed compaction, branch summaries, and bug-report summaries in SDK sessions passing the stored OAuth token back to the model runtime as an explicit API key. Provider auth that treats explicit keys differently from stored OAuth now sees the OAuth login, as normal prompts do ([#11520](https://github.com/earendil-works/pi/issues/11520)) - Fixed a redundant system prompt update (tools, rules) when an extension set the same active tools in a different order, for example a reversed `pi.setActiveTools()` list. Tools that stay active now keep their order and newly activated tools are appended ([#11495](https://github.com/earendil-works/pi/issues/11495)) +- Fixed `pi.exec()` crashing pi with `RangeError: Invalid string length` when a command printed more output than fits in a string. The default output cap is the smaller of 256 MiB and the runtime string limit; past the cap, the command is terminated and the result has `truncated: true` and a nonzero exit code ([#11360](https://github.com/earendil-works/pi/issues/11360)) +- Fixed `pi.exec()` never returning when a command ignored SIGTERM after a timeout or abort. pi now sends SIGKILL 5 seconds later, as intended - Fixed `pi install` and `pi remove` adding ever deeper nested entries to the npm `package-lock.json` when the agent directory is reached through a symlink, such as `~/.pi` linked into a dotfiles repo. pi now passes the resolved real path to npm ([#11358](https://github.com/earendil-works/pi/issues/11358)) - MCP sign-in now rejects authorization endpoints and other OAuth metadata URLs that are not `http:` or `https:`. Before, a malicious MCP server could supply a `file:` or custom protocol URL, which pi passed to the platform URL handler. - Automatic `fd` and `rg` downloads no longer fetch x86-64 binaries on ARM32 and other architectures without a prebuilt release. pi now warns and recommends installing the tool with the system package manager ([#11209](https://github.com/earendil-works/pi/issues/11209)) @@ -23,10 +60,67 @@ - Fixed HTML export showing a blank page for sessions with thousands of entries ([#11225](https://github.com/earendil-works/pi/issues/11225), [#11293](https://github.com/earendil-works/pi/pull/11293)) - Fixed HTML export not showing system messages: the system prompt and tools, and later section updates such as after `/reload`. `pi --export` now includes the system prompt and tools too ([#11261](https://github.com/earendil-works/pi/issues/11261), [#3464](https://github.com/earendil-works/pi/issues/3464), [#11290](https://github.com/earendil-works/pi/pull/11290)) - Fixed aborting a bash command on macOS sometimes running the next command of a list such as `long_step; next_step`. pi now stops the process group before killing it ([#11252](https://github.com/earendil-works/pi/issues/11252), [#11289](https://github.com/earendil-works/pi/pull/11289)) -- Fixed streamed tool calls re-running their call renderer on every argument delta; the interactive mode now renders the latest arguments once per frame, so a large streamed `write` no longer re-highlights its whole content per delta ([#9265](https://github.com/earendil-works/pi/issues/9265)) +- Fixed large streamed tool calls taking excessive CPU time and memory because every delta re-parsed the accumulated arguments. Providers and agent proxy streams now parse each delta once, grammar tools no longer copy their entire input per delta, and interactive tool renderers run at most once per frame ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed the login dialog opening sign-in URLs that are not web URLs, and printing control characters from sign-in, device-code, and info URLs, which could end a terminal hyperlink and send escape sequences to the terminal - Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored in slash command and file autocomplete, `/settings`, and the model, scoped models, login, fork, `/mcp`, and llama.cpp selectors. In fullscreen mode, PageUp/PageDown now move an open list or autocomplete menu instead of scrolling the transcript, and a page key bound to Ctrl+V pages autocomplete instead of pasting an image ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) - Fixed tool output and `!` command output showing the parameters of control sequences with a `<`, `=` or `>` prefix as text, for example `35;59;1M` from a mouse report `ESC[<35;59;1M` or `0;276;0c` from a terminal reply `ESC[>0;276;0c`. Text after the keypad-mode sequences `ESC=` and `ESC>` is still kept ([#11377](https://github.com/earendil-works/pi/issues/11377)) +- Fixed `pi -e npm:` staying on the version from the first download. On each start and `/reload`, pi now reinstalls an `-e` npm package without an exact version when the registry has a newer matching version. `pi -e git:@` likewise stayed on its first checkout and is now fetched and reset to the branch; a commit-hash ref that is already checked out is not fetched. Offline mode and failed checks keep the cached copy ([#11375](https://github.com/earendil-works/pi/issues/11375), [#11590](https://github.com/earendil-works/pi/pull/11590)) +- Fixed Windows paths being compared case-sensitively. When the working directory and home directory differed only in drive-letter casing (`c:\Users\me` vs `C:\Users\me`), skills in `~/.agents/skills` loaded twice, as project and user skills, and were reported as name collisions. The same mismatch also re-asked for saved project trust, hid sessions in a custom session directory from `--continue` and `/resume`, and stopped `pi remove` from matching configured local packages ([#10488](https://github.com/earendil-works/pi/issues/10488)) +- Fixed paths next to the home directory being shown with a broken `~` prefix, for example `/home/me2/project` shown as `~2/project` when home is `/home/me` +- Fixed `pi mcp login -l` and `pi mcp logout -l` failing with `Unknown option --local.` although `pi mcp add` and `pi mcp remove` accept `-l`. With `-l`, they now require the server to be a project server. Unknown options are now reported as typed, and a mistyped server name gets a suggestion, for example `Did you mean "gurker"?` +- Fixed `pi update` naming pnpm for installs that only live under `PNPM_HOME`, for example a wrapper under `$PNPM_HOME/global-nub/`. pi said "not managed by a global pnpm install"; it now names the package manager only when that manager owns the install ([#7756](https://github.com/earendil-works/pi/issues/7756)) +- Fixed image paste, copy, and opening the browser for sign-in failing silently on WSL when the working directory is deep. Windows programs such as `powershell.exe` and `explorer.exe` cannot start from a directory whose `\\wsl.localhost\...` path exceeds 260 characters; pi now starts these helpers from `/` ([#11592](https://github.com/earendil-works/pi/issues/11592)) +- Fixed the external editor failing silently, for example when the editor command is wrong or a Windows editor cannot start from a deep WSL directory. pi now shows a status message such as `External editor exited with code 1; prompt unchanged` ([#11592](https://github.com/earendil-works/pi/issues/11592)) +- Fixed `tool_search` and codemode `searchTools()` ranking a tool below tools that only mention its name in their descriptions, for example `peekaboo app` ranking `mcp__peekaboo__app` 14th. A tool matching more of the query's words now ranks above one matching fewer, a query word in a tool's name scores more than any number of mentions in its description, a tool whose whole name is in the query gets a large bonus, and long server instructions no longer lower the scores of that server's tools ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and codemode `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) +- Fixed `pi update` stopping with `git clean -fdx failed with code 1` on Windows when another pi session had a git extension's native module or executable open. Windows cannot delete such files, so the update left the new code with the old dependencies. pi now warns, installs the dependencies anyway, and retries the clean on the next update ([#11597](https://github.com/earendil-works/pi/issues/11597)) +- Fixed tool rows and streamed answers highlighting the same code again on every redraw. A running codemode row highlighted its whole script on every progress update and timer tick, a complete `write` row its whole file on every update and expand toggle, a streamed `write` its first 50 lines on every delta, and a streamed answer every earlier code block on every delta. Rows now reuse their last highlighting until the code, theme, or loaded grammars change, and assistant messages keep their rendered Markdown blocks while streaming ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed resized photos being sent as multi-megabyte PNGs even when JPEG was much smaller, causing large requests to fail or time out. Image resizing now tries the smaller encoding first ([#11551](https://github.com/earendil-works/pi/issues/11551)) +- Fixed `pi update --extensions` reporting success without warning when npm's stale registry metadata left a package at its old version. pi now reports the installed and expected versions ([#11536](https://github.com/earendil-works/pi/issues/11536)) +- Fixed skill validation warnings being labeled `[Skill conflicts]` even when there was no name collision. They are now shown under `[Skill warnings]` ([#11521](https://github.com/earendil-works/pi/issues/11521)) +- Fixed codemode tool descriptions including entire union and multi-line output types in every request. Object unions now summarize their fields and discriminators; other multi-line types point to `describeTool()` for the full declaration ([#11501](https://github.com/earendil-works/pi/issues/11501)) +- Fixed the model-scope startup notice being left in the shell after exiting fullscreen mode. It now appears inside pi's fullscreen header ([#11508](https://github.com/earendil-works/pi/issues/11508)) +- Fixed startup from a deleted working directory crashing with an uncaught `uv_cwd` stack trace. pi now asks the user to change to an existing directory ([#11509](https://github.com/earendil-works/pi/issues/11509)) +- Fixed failed session writes leaving entries in memory that were absent from disk, so later entries pointed to missing parents and resumed sessions lost earlier history and tools ([#11460](https://github.com/earendil-works/pi/issues/11460)) +- Fixed `/tree` branch summaries ignoring the session's thinking level, which could make reasoning models reject summary requests with an unsupported `none` effort ([#11444](https://github.com/earendil-works/pi/issues/11444)) +- Fixed hidden thinking with an empty hidden-thinking label still adding blank rows before answers and tool calls ([#11402](https://github.com/earendil-works/pi/issues/11402)) +- Fixed package installation and removal printing npm audit and funding advice, including `npm audit fix` instructions that could install a second copy of pi into the package directory ([#11409](https://github.com/earendil-works/pi/issues/11409)) +- Fixed replacing an extension editor while a dialog was open hiding the dialog and leaving it unanswerable. The new editor now appears after the dialog closes ([#11431](https://github.com/earendil-works/pi/issues/11431)) +- Fixed `grep` context lines being offset from ripgrep's match line numbers when a file contains a lone carriage return ([#11398](https://github.com/earendil-works/pi/issues/11398)) +- Fixed `--model /` failing for provider IDs containing slashes, such as URL-derived llama.cpp provider IDs ([#11394](https://github.com/earendil-works/pi/issues/11394)) +- Fixed slow session forking on filesystems where opening files is expensive by writing the fork through one file handle ([#11253](https://github.com/earendil-works/pi/issues/11253)) +- Fixed `pi.exec()` corrupting UTF-8 characters split across stdout or stderr chunks ([#11312](https://github.com/earendil-works/pi/issues/11312)) +- Fixed `pi.exec()` returning empty stderr when a command could not be spawned, such as a missing executable. It now preserves the spawn error message ([#11257](https://github.com/earendil-works/pi/pull/11257) by [@muyiyr](https://github.com/muyiyr)) +- Fixed a configured package failing to install crashing startup. pi now skips the package, reports an extension warning, and retries on the next start or `/reload`; explicit install and update commands still fail with a nonzero exit ([#11304](https://github.com/earendil-works/pi/pull/11304) by [@Lincoln504](https://github.com/Lincoln504)) +- Fixed the todo extension example crashing the TUI when a tool call failed argument validation ([#8057](https://github.com/earendil-works/pi/pull/8057) by [@cyzlmh](https://github.com/cyzlmh)) +- Updated `undici` to 8.12.3 and the Gondolin extension example to 0.14.0 for security advisories +- Fixed OpenRouter cost totals using catalog estimates instead of reported billing for Chat Completions and Anthropic Messages. BYOK totals include both OpenRouter charges and reported upstream costs; component costs remain catalog estimates ([#9980](https://github.com/earendil-works/pi/issues/9980)) +- Fixed OpenCode Zen and OpenCode Go requests using shared endpoints instead of their protocol-specific OpenAI, Anthropic, Google, and System One endpoints ([#11434](https://github.com/earendil-works/pi/pull/11434) by [@vimtor](https://github.com/vimtor)) +- Fixed Zhipu/Z.ai usage-limit errors (code `1308`) being treated as transient HTTP 429 errors, causing automatic retries that cannot succeed until the usage cap resets ([#11537](https://github.com/earendil-works/pi/issues/11537)) +- Fixed LiteLLM's `ContextWindowExceededError` not being recognized as context overflow ([#11492](https://github.com/earendil-works/pi/issues/11492)) +- Fixed Gemini tool schemas sending unsupported `const` keywords. They are now converted to single-value `enum` schemas without rewriting literal data in defaults, examples, or enum values, or losing properties named `__proto__` ([#11308](https://github.com/earendil-works/pi/issues/11308)) +- Fixed the faux provider counting simulated cache-write tokens as both input and cache writes +- Fixed Orca terminals falling back to text for inline images. Orca is now recognized as supporting true color and hyperlinks, and Kitty images when it advertises `ORCA_IMAGE_PROTOCOL=kitty` ([#9329](https://github.com/earendil-works/pi/pull/9329) by [@XRX193](https://github.com/XRX193)) +- Fixed bare Markdown URLs swallowing following CJK punctuation and text; explicit Markdown links and angle autolinks are unchanged ([#10934](https://github.com/earendil-works/pi/pull/10934) by [@haoqixu](https://github.com/haoqixu)) +- Fixed Kitty-protocol F1–F12 shortcuts, including modified function keys, not being recognized, and F13+ and other functional keys inserting invisible private-use characters into the editor ([#11419](https://github.com/earendil-works/pi/issues/11419), [#11345](https://github.com/earendil-works/pi/issues/11345)) +- Fixed Kitty images inside padded or colored boxes disappearing when their first row scrolled above the fullscreen viewport. Partially visible images are now cropped correctly, including beside other text +- Fixed deeply nested Markdown blockquotes producing exponentially many lines, hanging or crashing rendering once their borders exhausted the available width ([#11272](https://github.com/earendil-works/pi/issues/11272)) +- Fixed Claude sessions failing with `role 'system' must precede an 'assistant' message` when a system update was followed by an empty or reasoning-only assistant reply ([#11091](https://github.com/earendil-works/pi/issues/11091)) +- Fixed Claude mid-conversation thinking-level changes failing with `Invalid effort level` on accounts whose organization caps effort below `high` ([#10063](https://github.com/earendil-works/pi/issues/10063)) +- Fixed Kimi K3.1 Pro models using Kimi K2 request settings on Kimi Coding, Moonshot, and OpenCode Go. They now send the correct reasoning effort, preserve earlier thinking, accept tools added mid-conversation, and use K3-family pricing +- Fixed OpenCode DeepSeek V4.1 Flash requests failing when long cache retention sent the unsupported `prompt_cache_retention` parameter +- Fixed GPT-5.4 and GPT-5.5 failing with `Missing namespace for function_call` after switching from a model that had called tools ([#11415](https://github.com/earendil-works/pi/issues/11415)) +- Fixed Bedrock Claude requests failing when preferred strict tools exceed the provider's strict-tool limits. Requests now retry once with preferred tools non-strict while keeping required strict tools strict ([#11416](https://github.com/earendil-works/pi/issues/11416)) +- Fixed OpenRouter's `~anthropic/claude-*-latest` aliases losing earlier thinking after tool calls by using the Anthropic Messages API ([#9534](https://github.com/earendil-works/pi/issues/9534)) +- Fixed terminal titles containing control characters being able to inject terminal escape sequences, and pi-messages streams accepting invalid content-block indices +- Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`; word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) +- Fixed display math wider than the content area being split apart by line wrapping; it now falls back to raw LaTeX ([#11483](https://github.com/earendil-works/pi/issues/11483)) +- Fixed styled words ending exactly at the wrap width adding a leading space on the next line, and strikethrough leaking into wrapped-line padding ([#10925](https://github.com/earendil-works/pi/issues/10925)) +- Fixed `/tree` branch summaries including failed assistant replies and original content that context edits had hidden or replaced. Summaries now honor the same context edits as live requests and compaction ([#11602](https://github.com/earendil-works/pi/issues/11602)) + +### Removed + +- Removed the native OS `sandbox/` extension example and its dependency. Use the [Gondolin example](examples/extensions/gondolin/) or the [containerization guide](docs/containerization.md). ## [1.4.0] - 2026-10-21 diff --git a/packages/coding-agent/docs/cli.md b/packages/coding-agent/docs/cli.md index c471f7576..8861d22ad 100644 --- a/packages/coding-agent/docs/cli.md +++ b/packages/coding-agent/docs/cli.md @@ -84,15 +84,15 @@ pi --continue See [Sessions and Context](sessions.md) for resuming, forking, naming, and storing sessions. - `-c`, `--continue`
- Continues the most recent session for the current project. + Continues the most recent session for the current directory. - `-r`, `--resume`
- Opens the session selector. + Opens the session selector and resumes the chosen session in its working directory. Press Tab to include sessions from other directories. - `--session `
- Opens by file path, exact ID, or partial ID. Pi searches the current project first and offers to fork a cross-project match. + Opens a session by file path or full/partial ID, and resumes it in its working directory. If the ID only matches sessions stored for another directory, Pi offers to fork it into the current directory instead. - `--session-id `
- Opens the exact project session ID or creates it if absent. IDs accept letters, numbers, `.`, `_`, and `-`. + Opens the current directory's existing session with the ID, or creates it otherwise. IDs accept letters, numbers, `.`, `_`, and `-`. - `--fork `
- Forks an existing session into a new session for the current project. + Forks a session, chosen by file path or full/partial ID, into a new session in the current directory. - `--session-dir `
Overrides storage and lookup. It takes precedence over `PI_CODING_AGENT_SESSION_DIR` and the `sessionDir` setting. - `--no-session`
@@ -282,6 +282,16 @@ Add `--force` to reinstall Pi when the selected update includes Pi. `pi update` cannot update Pi when another package manager provides it, such as Nix. Update Pi with that package manager, for example `nix profile upgrade pi`. Package and model catalog updates still work. +#### Custom update command + +If a wrapper or another package manager provides Pi, set `PI_SELF_UPDATE_COMMAND` so `pi update` runs its update command: + +```sh +export PI_SELF_UPDATE_COMMAND="mise use -g npm:{packageName}@{version}" +``` + +`pi update` runs it only when pi.dev has a newer release, replacing `{packageName}`, `{version}`, and `{installSpec}` (`{packageName}@{version}`) with that release. The value is split on spaces. For arguments with spaces, use the `selfUpdateCommand` setting in `~/.pi/agent/settings.json` instead, an argv array such as `["nix", "profile", "upgrade", "pi"]`. + ### Aliases and command options - `pi uninstall ` is an alias for `pi remove `. @@ -322,11 +332,11 @@ These commands work outside a session, so agents can run them through `bash`. Se | Command | Description | |---|---| | `pi mcp add [options] -- [args...]` | Add or replace a stdio server in `mcp.json`; `--env KEY=VALUE` (repeatable) and `--cwd ` set its environment and working directory. Arguments after the command are passed to it | -| `pi mcp add [options] --url ` | Add or replace a streamable HTTP server; `--header KEY=VALUE` (repeatable), `--bearer-token-env-var ` (sends `Authorization: Bearer ${NAME}`), `--oauth-client-id`, `--oauth-client-secret`, `--oauth-callback-port`, and `--oauth-client-name` configure authentication | +| `pi mcp add [options] --url ` | Add or replace a streamable HTTP server; `--header KEY=VALUE` (repeatable), `--bearer-token-env-var ` (sends `Authorization: Bearer ${NAME}`), `--oauth-client-id`, `--oauth-client-secret`, `--oauth-callback-port`, `--oauth-client-name`, and `--oauth-client-registration` (`auto`, `cimd`, or `dcr`) configure authentication | | `pi mcp remove ` | Remove a server from `mcp.json`; stored OAuth credentials are kept | | `pi mcp list [--json]` | Connect to every enabled server and print its state, tools, and errors; exit with `1` when a config entry is invalid or an enabled server is not connected | -| `pi mcp login [--timeout ]` | Sign in to an OAuth server: open the authorization page and wait for the browser; a terminal also accepts the pasted redirect URL. `--timeout` limits the whole sign-in, including requests to the authorization server (default 300 seconds) | -| `pi mcp logout ` | Delete the stored OAuth credentials of a server | +| `pi mcp login [-l] [--timeout ]` | Sign in to an OAuth server: open the authorization page and wait for the browser; a terminal also accepts the pasted redirect URL. `--timeout` limits the whole sign-in, including requests to the authorization server (default 300 seconds). `-l` requires the server to be defined in the project's `.pi/mcp.json` | +| `pi mcp logout [-l]` | Delete the stored OAuth credentials of a server | `add` and `remove` change `~/.pi/agent/mcp.json`, or `.pi/mcp.json` in the current directory with `--local` (`-l`). `add` also takes `--exposure ` (see [Exposure](mcp.md#control-tool-exposure)) and `--description ` and does not connect; run `pi mcp list` to check the server. diff --git a/packages/coding-agent/docs/environment-variables.md b/packages/coding-agent/docs/environment-variables.md index b485d6051..14dd3e142 100644 --- a/packages/coding-agent/docs/environment-variables.md +++ b/packages/coding-agent/docs/environment-variables.md @@ -83,6 +83,7 @@ These variables are read by Pi itself: | `PI_PACKAGE_DIR` | Override the package directory, useful for Nix/Guix store paths | | `PI_OFFLINE` | Disable automatic network activity, including model catalog refreshes | | `PI_SKIP_VERSION_CHECK` | Disable the `pi.dev` latest-version request | +| `PI_SELF_UPDATE_COMMAND` | Command `pi update` runs to update Pi itself, split on whitespace, with `{packageName}`, `{version}`, and `{installSpec}` placeholders. Takes precedence over the `selfUpdateCommand` setting. See [Custom update command](cli.md#custom-update-command) | | `PI_TELEMETRY` | Override install/update telemetry and provider attribution headers: `1`/`true`/`yes` or `0`/`false`/`no` | | `PI_CODEMODE_MAX_WORKERS` | Maximum codemode workers alive at once across all sessions in the process; default `2`. Each worker's script may use up to 256 MB. Raise it when one process runs several sessions, such as in-process subagents. Further scripts wait for a free worker. See [Codemode](codemode.md) | | `PI_CACHE_RETENTION` | Set to `long` for extended provider prompt caching where supported | diff --git a/packages/coding-agent/docs/extensions.md b/packages/coding-agent/docs/extensions.md index 5a815072b..6fea5a841 100644 --- a/packages/coding-agent/docs/extensions.md +++ b/packages/coding-agent/docs/extensions.md @@ -159,7 +159,7 @@ Settings also accept built-in extensions as `builtin:` entries; see [Setti | `@earendil-works/pi-coding-agent` | Extension types (`ExtensionAPI`, `ExtensionContext`, events) and helpers | | `@earendil-works/pi-agent-core` | Agent message and tool types | | `typebox`, `typebox/compile`, `typebox/value` | Schema definitions for tool parameters | -| `@earendil-works/pi-ai` | AI utilities (`StringEnum` for string enums, transcript helpers, model types) | +| `@earendil-works/pi-ai` | AI utilities (`StringEnum` for string enums, transcript helpers, model types). Inside Pi it also exports TypeBox's `Type`, `Static` and `TSchema`, which the published package no longer re-exports; import them from `typebox` in new code | | `@earendil-works/pi-ai/compat`, `/models`, `/oauth`, `/providers/all` | Provider construction, built-in APIs, and OAuth helpers | | `@earendil-works/pi-tui` | TUI components for custom rendering | @@ -169,6 +169,18 @@ npm dependencies work too. Add a `package.json` next to your extension (or in a For distributed Pi packages installed with `pi install` (npm or git), runtime deps must be in `dependencies`; `devDependencies` are not installed. Do not list the host-provided packages above in `dependencies`. See [Declare dependencies](packages.md#declare-dependencies). +### Tool parameter schemas and TypeBox + +Pi validates tool arguments as JSON Schema with `@earendil-works/pi-schema`, whatever built the schema; TypeBox schemas are JSON Schema and work unchanged. Before validation, loosely typed arguments are coerced: numeric strings to numbers, `"true"` to `true`, a single value to a one-item array, `null` in an optional property dropped. TypeBox features outside JSON Schema do not apply to tool arguments: + +- `Type.Refine()` refinements are not run. Check such constraints in `execute()`. +- `Type.BigInt()`, `Type.Undefined()` and `Type.Void()` reject every argument, since JSON has no such values. Codecs (`Type.Codec()`) do not decode: `execute()` receives the JSON value. +- Formats registered with TypeBox's `Format.Set()` are not checked. The standard formats are, plus `url` and `json-pointer-uri-fragment`. +- Property names are matched as names, not as regular expressions, and `additionalProperties` applies only to properties the object does not declare. +- A `$ref` that names nothing rejects only arguments at its place. + +Validation errors the model sees use pi-schema's messages, such as `- path: missing` or `- limit: expected number, got string`. + Node.js built-ins (`node:fs`, `node:path`, etc.) are also available. ## Writing an Extension @@ -1870,13 +1882,15 @@ pi.on("session_start", () => { ### pi.exec(command, args, options?) -Execute a shell command. +Execute a command without a shell. ```typescript const result = await pi.exec("git", ["status"], { signal, timeout: 5000 }); -// result.stdout, result.stderr, result.code, result.killed +// result.stdout, result.stderr, result.code, result.killed, result.truncated ``` +Options: `signal` and `timeout` terminate the command, `cwd` sets the working directory, and `maxBuffer` limits stdout and stderr combined (default 256 MiB). When output exceeds `maxBuffer`, pi terminates the command and returns the output up to the limit with `truncated: true`, `killed: true`, and a nonzero `code`. `killed` is true whenever pi terminated the command: timeout, abort, or `maxBuffer`. A command that ignores SIGTERM gets SIGKILL after 5 seconds. + ### pi.getActiveTools() / pi.getAllTools() / pi.setActiveTools(names) Manage active tools. This works for both built-in tools and dynamically registered tools. `pi.getActiveTools()` returns the active tool names as `string[]`; `pi.getAllTools()` returns metadata for all configured tools. @@ -2520,6 +2534,7 @@ pi.registerTool({ - `toolCallId`, `cwd`, `executionStarted`, `argsComplete`, `isPartial`, `expanded`, `showImages`, `isError` - `durationMs` - how long execution took, from the final result; `undefined` while running - `outputPad` - horizontal padding from the `outputPad` setting; renderers with `renderShell: "self"` apply it themselves +- `keyText()`, `keyHint()` - format the user's configured keybindings, see [Keybinding Hints](#keybinding-hints) Use `context.state` for cross-slot shared state. Keep slot-local caches on the returned component instance when you want to reuse and mutate the same component across renders. @@ -2570,24 +2585,24 @@ If a slot intentionally has no visible content, return an empty `Component` such #### Keybinding Hints -Use `keyHint()` to display keybinding hints that respect the active keybinding configuration: +Use `keyHint()` and `keyText()` from the render context to display keybinding hints that respect the user's keybinding configuration: ```typescript -import { keyHint } from "@earendil-works/pi-coding-agent"; - renderResult(result, { expanded }, theme, context) { let text = theme.fg("success", "✓ Done"); if (!expanded) { - text += ` (${keyHint("app.tools.expand", "to expand")})`; + text += ` (${context.keyHint("app.tools.expand", "to expand")})`; } return new Text(text, 0, 0); } ``` -Available functions: -- `keyHint(keybinding, description)` - Formats a configured keybinding id such as `"app.tools.expand"` or `"tui.select.confirm"` -- `keyText(keybinding)` - Returns the raw configured key text for a keybinding id -- `rawKeyHint(key, description)` - Format a raw key string +The same functions are available as `options.keyHint()`/`options.keyText()` in message and entry renderers: + +- `keyHint(keybinding, description)` - Formats a configured keybinding id such as `"app.tools.expand"` or `"tui.select.confirm"`, followed by the description +- `keyText(keybinding)` - Returns the raw configured key text for a keybinding id, or `""` if the id has no keys + +In renderers, use these instead of importing `keyHint()` or `keyText()` from `@earendil-works/pi-coding-agent`. An extension can resolve its own copy of the package, for example from its own `node_modules`, and that copy does not see pi's configured keybindings: `app.*` ids then format as `""`. `rawKeyHint(key, description)` formats a literal key string and is safe to import. Use namespaced keybinding ids: - Coding-agent ids use the `app.*` namespace, for example `app.tools.expand`, `app.editor.external`, `app.session.rename` @@ -3144,6 +3159,8 @@ pi.registerMessageRenderer("my-extension", (message, options, theme) => { if (expanded && message.details) { text += "\n" + theme.fg("dim", JSON.stringify(message.details, null, 2)); + } else if (message.details) { + text += ` (${options.keyHint("app.tools.expand", "to expand")})`; } return new Text(text, outputPad, 0); @@ -3307,7 +3324,6 @@ All examples in [examples/extensions/](../examples/extensions/). | **Remote & Sandbox** ||| | `ssh.ts` | SSH remote execution | `registerFlag`, `on("user_bash")`, `on("before_agent_start")`, tool operations | | `interactive-shell.ts` | Persistent shell session | `on("user_bash")` | -| `sandbox/` | Sandboxed tool execution | Tool operations | | `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | Tool operations, built-in tool overrides, `on("user_bash")` | | `subagent/` | Spawn sub-agents | `registerTool`, `exec` | | **Games** ||| diff --git a/packages/coding-agent/docs/mcp.md b/packages/coding-agent/docs/mcp.md index 5d5eb338e..5c6aa7d86 100644 --- a/packages/coding-agent/docs/mcp.md +++ b/packages/coding-agent/docs/mcp.md @@ -126,7 +126,7 @@ Remote servers that use OAuth, such as Sentry, need no credentials in `mcp.json` When the server rejects an unauthenticated connection, `/mcp` shows that it needs sign-in. Select "Sign in", run `/mcp login sentry`, or run `pi mcp login sentry`. Pi opens the authorization page and waits for approval. If the browser runs on another machine, such as over SSH, paste its redirected URL into the sign-in screen. A running session uses the new credentials on its next turn. -Pi registers itself with the authorization server, stores tokens in `~/.pi/agent/mcp-auth.json`, and refreshes access tokens when they expire or the server rejects them. If a server later requests additional scope, Pi asks for sign-in again. Signing out deletes the stored credentials. +Pi identifies itself to the authorization server with its Client ID Metadata Document or by registering a client, stores tokens in `~/.pi/agent/mcp-auth.json`, and refreshes access tokens when they expire or the server rejects them. If a server later requests additional scope, Pi asks for sign-in again. Signing out deletes the stored credentials. Credentials belong to a server name and URL. Servers with the same URL under different names, such as one per account, sign in separately; servers with the same name and URL in different `mcp.json` files share one sign-in. @@ -159,17 +159,19 @@ Pi registers as `pi`. Some servers only accept registrations from known clients. The name is only sent when Pi registers a client. To register again under a new name, sign out first. -Some authorization servers allow clients by their Client ID Metadata Document URL instead of registering them. Set `clientRegistration` to `cimd` to identify as Pi's document on pi.dev instead of registering: +Without `clientId`, Pi identifies itself with its Client ID Metadata Document on pi.dev when the authorization server advertises support for documents and public clients. Otherwise it registers a client dynamically. A client registered before stays in use while its redirect URI matches, until you sign out. With the document, the client ID is `https://pi.dev/oauth/client.json` with the redirect URI `http://127.0.0.1:/callback`. If the authorization server does not send the `iss` parameter in authorization responses (RFC 9207), Pi uses a document and redirect path specific to the MCP server instead: `https://pi.dev/oauth//client.json` with `http://127.0.0.1:/callback/`. + +Pi registers instead of using the document when `clientName` is set or `callbackUrl` is not `/callback` on `localhost` or `127.0.0.1`. Set `clientRegistration` to choose the method: `auto` (default) as described, `dcr` to always register, or `cimd` to always use the document. With `cimd`, sign-in fails if the authorization server does not support documents for public clients. Use `dcr` when the authorization page reports an invalid client: ```json { "mcpServers": { - "example": { "url": "https://mcp.example.com/mcp", "oauth": { "clientRegistration": "cimd" } } + "example": { "url": "https://mcp.example.com/mcp", "oauth": { "clientRegistration": "dcr" } } } } ``` -The client ID is `https://pi.dev/oauth/client.json` with the redirect URI `http://127.0.0.1:/callback`. If the authorization server does not send the `iss` parameter in authorization responses (RFC 9207), Pi uses a document and redirect path specific to the MCP server instead: `https://pi.dev/oauth//client.json` with `http://127.0.0.1:/callback/`. The authorization server must advertise Client ID Metadata Document support and public clients, or sign-in fails. `cimd` cannot be combined with `clientId` or `clientName`, and a `callbackUrl` must use `localhost` or `127.0.0.1` with the path `/callback`. +`pi mcp add --oauth-client-registration ` sets it when adding a server. `cimd` cannot be combined with `clientId` or `clientName`, and a `callbackUrl` must use `localhost` or `127.0.0.1` with the path `/callback`. Pi finds the authorization server through the server's protected resource metadata (RFC 9728) and checks that the authorization server's metadata names the expected issuer (RFC 8414). Some servers advertise the wrong authorization server or none, so sign-in opens a page that does not exist. Set `authServerMetadataUrl` to the metadata document of the right authorization server: diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index e076ac139..803458a8b 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -141,7 +141,7 @@ Compatibility settings should describe verified differences in the endpoint's re ## Use classifier models -Classifier models do not chat. They answer typed questions about JSON state: pick one of several choices, answer yes or no, or give a score, each with probabilities. Pi includes TypeSafe's Jev model from these providers, Cloudflare's Clef, Clef Flash, and Clef Omni models from Workers AI and OpenRouter, and OpenAI's GPT-6 Luna through the [Decisions API](https://developers.openai.com/api/docs/guides/decisions): +Classifier models do not chat. They answer typed questions about JSON state: pick one of several choices, answer yes or no, or give a score, each with probabilities. Pi includes TypeSafe's Jev model from these providers, Cloudflare's Clef, Clef Flash, and Clef Omni models from Workers AI and OpenRouter, OpenAI's GPT-6 Luna through the [Decisions API](https://developers.openai.com/api/docs/guides/decisions), and Microsoft-Decision-1 through GitHub Copilot: | Provider | Model IDs | Authentication | |---|---|---| @@ -151,6 +151,7 @@ Classifier models do not chat. They answer typed questions about JSON state: pic | `vercel-ai-gateway` | `typesafe-ai/jev` | `AI_GATEWAY_API_KEY` | | `opencode` | `jev-1.13`, `jev-1.13-free` | `OPENCODE_API_KEY` | | `openai` | `gpt-6-luna` | `OPENAI_API_KEY` | +| `github-copilot` | `microsoft-decision-1` | `COPILOT_GITHUB_TOKEN` or `/login` | Chat models on a [llama.cpp router](llama-cpp.md#classification) are also listed as classifier models. diff --git a/packages/coding-agent/docs/packages.md b/packages/coding-agent/docs/packages.md index 6cf306318..6aa1d8afc 100644 --- a/packages/coding-agent/docs/packages.md +++ b/packages/coding-agent/docs/packages.md @@ -28,6 +28,8 @@ Use `--extension` or `-e` to try a package for one invocation without adding it pi -e npm:@example/pi-tools ``` +Pi caches `-e` packages under `~/.pi/agent/tmp/extensions/` and checks them on every start and `/reload`. An npm package is reinstalled when the registry has a newer version that matches the specification. A version pinned to an exact number, such as `@1.2.3`, is not checked. A git package is fetched and reset to the remote branch or ref. A ref that is a commit hash already checked out is not fetched. If the check fails or offline mode is on, Pi loads the cached copy. + ## Choose a source | Source | Example | Behavior | diff --git a/packages/coding-agent/docs/sessions.md b/packages/coding-agent/docs/sessions.md index 1a8be36c5..aeb59a308 100644 --- a/packages/coding-agent/docs/sessions.md +++ b/packages/coding-agent/docs/sessions.md @@ -11,7 +11,17 @@ pi --continue pi --resume ``` -`--continue` opens the most recent session for the current working directory. `--resume` opens the session picker. In interactive mode, `/resume` opens the same picker and `/new` starts a new session. +`--continue` opens the most recent session for the current directory. `--resume` opens the session picker. In interactive mode, `/resume` opens the same picker and `/new` starts a new session. + +Each session records the working directory it was created in and never changes it. Pi runs a session in its working directory, so tools, `@path` references, project configuration, and context files resolve from there. + +| Entry point | Working directory | +|---|---| +| `--continue` | The current directory | +| `--resume`, `/resume`, `--session`, `/import` | The session's working directory, which can differ from the current directory | +| `--fork`, or a `--session` ID from another directory that you choose to fork | The current directory, in a new session | + +If a session's working directory no longer exists, interactive mode offers to run the session in the current directory instead. Pi does not store that choice, so it asks again the next time you open the session. Other modes exit with an error. Use `/name` or `--name` to assign a recognizable session name. Run `/session` to verify the current session file, ID, message count, token usage, and cost. @@ -51,8 +61,6 @@ By default, Pi stores sessions under `~/.pi/agent/sessions/`, grouped by working Use `--no-session` for an ephemeral run. An ephemeral session cannot be resumed after Pi exits. -Use `--session` when you already know the session path or ID. Use `--fork` to create a new session from an existing session before interactive mode starts. - ## Export or share a session Use `/export` to write the current session as HTML or JSONL. Use `/share` to upload it and get a viewer link. By default, Pi uses a Radius artifact when Radius authentication is configured; otherwise, it uses a private GitHub gist. Set [`sessionShareDestination`](settings.md#sessions-and-context) to `gist` or `radius` to force one destination. diff --git a/packages/coding-agent/docs/settings.md b/packages/coding-agent/docs/settings.md index 538a61392..3de528c44 100644 --- a/packages/coding-agent/docs/settings.md +++ b/packages/coding-agent/docs/settings.md @@ -142,6 +142,7 @@ Keep `retry.provider.maxRetries` at `0` unless provider-level retries are requir | `shellPath` | string | Platform default | Custom shell executable path. Supports a leading `~`. | | `shellCommandPrefix` | string | None | Prefix prepended to every shell command. | | `npmCommand` | `string[]` | `npm` | Command and arguments used for npm package lookup and installation. | +| `selfUpdateCommand` | `string[]` | None | Command and arguments `pi update` runs to update Pi itself, with `{packageName}`, `{version}`, and `{installSpec}` placeholders. `PI_SELF_UPDATE_COMMAND` takes precedence. **Can only be set in agent-directory settings.** See [Custom update command](cli.md#custom-update-command). | See [Shell aliases](shell-aliases.md) for shell setup and [Pi Packages](packages.md) for package-manager behavior. diff --git a/packages/coding-agent/docs/slash-commands.md b/packages/coding-agent/docs/slash-commands.md index f2b5f3bac..cb534a363 100644 --- a/packages/coding-agent/docs/slash-commands.md +++ b/packages/coding-agent/docs/slash-commands.md @@ -21,14 +21,14 @@ Extensions, prompt templates, and skills can add commands. The command menu in P | Command | Description | |---|---| | `/new` | Start a new session | -| `/resume` | Switch to another saved session | +| `/resume` | Resume another session | | `/name [name]` | Set the session display name, or show the current name when omitted | | `/session` | Show current session information and statistics | | `/tree` | Navigate the session tree | | `/fork` | Create a new session from an earlier user message | | `/clone` | Duplicate the current session at its current position | | `/compact [instructions]` | Compact the current context, optionally with custom instructions | -| `/import ` | Import and resume a JSONL session | +| `/import ` | Import a JSONL session, preserving its ID and working directory | ## Export and share diff --git a/packages/coding-agent/docs/terminal-setup.md b/packages/coding-agent/docs/terminal-setup.md index a3c67af07..ab8e1dc0e 100644 --- a/packages/coding-agent/docs/terminal-setup.md +++ b/packages/coding-agent/docs/terminal-setup.md @@ -40,7 +40,7 @@ This is an iTerm2-wide setting and can also change native trackpad scrolling. Th ## Apple Terminal -Pi enables enhanced key reporting when available. If Terminal.app still sends plain Return for `Shift+Enter`, Pi uses a local macOS modifier fallback and treats it as `Shift+Enter`. +Pi enables enhanced key reporting when available. If Terminal.app still sends plain Return or `ESC` + Return for `Shift+Enter`, Pi uses a local macOS modifier fallback and treats it as `Shift+Enter`. `ESC` + Return without Shift held remains `Alt+Enter`. The fallback works only when Pi runs on the same Mac as Terminal.app. It cannot inspect the local modifier state when Pi runs on another machine over SSH. @@ -170,6 +170,8 @@ Windows Terminal uses Pi's Windows and WSL shortcut defaults. See [Keybindings]( ### Forward Shift+Enter +When Pi runs natively on Windows, it checks the local Shift key state and treats plain Return or `ESC` + Return as `Shift+Enter` while Shift is held. This fallback does not work in WSL or over SSH. If `Shift+Enter` still submits, forward it explicitly. + Open Windows Terminal's `settings.json` with `Ctrl+Shift+,` or **Settings > Open JSON file**. Add this object to its `actions` array: ```json diff --git a/packages/coding-agent/docs/tmux.md b/packages/coding-agent/docs/tmux.md index 091d61647..1a640d174 100644 --- a/packages/coding-agent/docs/tmux.md +++ b/packages/coding-agent/docs/tmux.md @@ -53,3 +53,9 @@ set -g extended-keys on Pi supports the xterm `modifyOtherKeys` format used by these versions. Restart tmux and repeat the verification steps. For older versions, upgrade tmux or use Pi outside tmux rather than relying on modified Enter shortcuts. + +## Known issues + +| Issue | Affects | Remedy | +|---|---|---| +| Text such as `c0c/0c0c/0c0c` appears in the editor at startup ([tmux#4749](https://github.com/tmux/tmux/issues/4749)) | tmux 3.6 | Upgrade tmux. Changing Pi's theme does not help. | diff --git a/packages/coding-agent/docs/windows.md b/packages/coding-agent/docs/windows.md index 26f2f961c..6e68979b8 100644 --- a/packages/coding-agent/docs/windows.md +++ b/packages/coding-agent/docs/windows.md @@ -76,6 +76,33 @@ JSON uses backslashes for escape sequences. When you write a Windows path with b See [Configure shell commands](shell-aliases.md) for command prefixes, aliases, and the complete shell-resolution behavior. +## Choose a terminal + +Windows terminals run Pi through a console host (conhost). Some bundle an up-to-date copy; others use the older one built into Windows, which has bugs that affect Pi. + +We recommend [Windows Terminal](https://aka.ms/terminal). It always ships an up-to-date conhost, and Pi is best tested with it. + +Last updated: October 2026. + +| Terminal | Conhost | Status | +|---|---|---| +| [Windows Terminal](https://aka.ms/terminal) | Bundled, current | Recommended | +| [VS Code](https://code.visualstudio.com), [Tabby](https://tabby.sh), [WezTerm](https://wezterm.org) nightly | Bundled, current | Works | +| [Alacritty](https://alacritty.org) | Built-in, unless `conpty.dll` and `OpenConsole.exe` from Windows Terminal are next to `alacritty.exe` | Works only with those files | +| WezTerm 20240203 and older | Bundled, old | [Known issues](#known-issues) | +| Classic console (`cmd.exe` or PowerShell without Windows Terminal), mintty (Git Bash), ConEmu, Cmder, [Hyper](https://hyper.is), Windows OpenSSH | Built-in | [Known issues](#known-issues) | + ## Configure Windows Terminal Windows Terminal reserves or rewrites some modified keys. See [Windows Terminal](terminal-setup.md#windows-terminal) to configure `Shift+Enter` and `Alt+Enter`, and [Keybindings](keybindings.md) for Pi's Windows and WSL shortcut defaults. + +## Known issues + +These issues come from conhost. Microsoft has fixed them in its source, but has not said when the built-in conhost will get the fixes. + +| Issue | Affects | Remedy | +|---|---|---| +| Switching from fullscreen to regular mode, or `Ctrl+G` from fullscreen, crashes the session ([#9186](https://github.com/earendil-works/pi/issues/9186), [microsoft/terminal#17515](https://github.com/microsoft/terminal/issues/17515)) | Built-in conhost | Use a terminal with a current conhost, or start Pi with `--tui-mode regular` | +| Text such as `;0;rgb:0000/0000/0000` appears in the prompt, and BEL opens the external editor ([#10256](https://github.com/earendil-works/pi/issues/10256)) | Built-in conhost behind another terminal: mintty, Windows OpenSSH | Use a terminal with a current conhost | +| Fragments such as `79` or `55` appear in the prompt ([#10717](https://github.com/earendil-works/pi/issues/10717)) | WezTerm 20240203 | Use WezTerm nightly | +| In regular mode, each keystroke redraws the input line on a new row ([#6300](https://github.com/earendil-works/pi/issues/6300)) | Built-in conhost, classic console | Use fullscreen mode (the default), or a terminal with a current conhost | diff --git a/packages/coding-agent/examples/extensions/README.md b/packages/coding-agent/examples/extensions/README.md index 7667ffe03..de24f7af4 100644 --- a/packages/coding-agent/examples/extensions/README.md +++ b/packages/coding-agent/examples/extensions/README.md @@ -23,9 +23,10 @@ cp permission-gate.ts ~/.pi/agent/extensions/ | `protected-paths.ts` | Blocks writes to protected paths (.env, .git/, node_modules/) | | `confirm-destructive.ts` | Confirms before destructive session actions (clear, switch, fork) | | `dirty-repo-guard.ts` | Prevents session changes with uncommitted git changes | -| `sandbox/` | OS-level sandboxing using `@anthropic-ai/sandbox-runtime` with per-project config | | `gondolin/` | Route built-in tools and `!` commands into a Gondolin micro-VM | +For isolation setup and limitations, see [Run Pi in an isolated environment](../../docs/containerization.md). + ### Custom Tools | Extension | Description | diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json index b7d2ebab0..c6ed32b21 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-custom-provider", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-custom-provider", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@anthropic-ai/sdk": "^0.52.0" } diff --git a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json index b22368863..b4900f974 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-anthropic/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-anthropic", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json index dbdc63c72..54e070434 100644 --- a/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json +++ b/packages/coding-agent/examples/extensions/custom-provider-gitlab-duo/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-custom-provider-gitlab-duo", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/debug-provider.ts b/packages/coding-agent/examples/extensions/debug-provider.ts index 14aae7027..2f13e0d24 100644 --- a/packages/coding-agent/examples/extensions/debug-provider.ts +++ b/packages/coding-agent/examples/extensions/debug-provider.ts @@ -6,7 +6,7 @@ * as expandable custom entries. */ -import { type ExtensionAPI, keyHint } from "@earendil-works/pi-coding-agent"; +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Box, Text } from "@earendil-works/pi-tui"; const ENTRY_TYPE = "debug-provider-events"; @@ -24,7 +24,7 @@ export default function (pi: ExtensionAPI) { let activeEvents: unknown[] | undefined; let completedEntry: ProviderDebugEntry | undefined; - pi.registerEntryRenderer(ENTRY_TYPE, (entry, { expanded }, theme) => { + pi.registerEntryRenderer(ENTRY_TYPE, (entry, { expanded, keyHint }, theme) => { const data = entry.data; if (!data) return new Text(theme.fg("warning", "[provider debug] Missing event data"), 0, 0); diff --git a/packages/coding-agent/examples/extensions/gondolin/package-lock.json b/packages/coding-agent/examples/extensions/gondolin/package-lock.json index 99a06ff69..0887734dd 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package-lock.json +++ b/packages/coding-agent/examples/extensions/gondolin/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-gondolin", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "@earendil-works/gondolin": "0.14.0" } diff --git a/packages/coding-agent/examples/extensions/gondolin/package.json b/packages/coding-agent/examples/extensions/gondolin/package.json index d19d7190f..762d1d230 100644 --- a/packages/coding-agent/examples/extensions/gondolin/package.json +++ b/packages/coding-agent/examples/extensions/gondolin/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-gondolin", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/extensions/sandbox/.gitignore b/packages/coding-agent/examples/extensions/sandbox/.gitignore deleted file mode 100644 index 3c3629e64..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/.gitignore +++ /dev/null @@ -1 +0,0 @@ -node_modules diff --git a/packages/coding-agent/examples/extensions/sandbox/index.ts b/packages/coding-agent/examples/extensions/sandbox/index.ts deleted file mode 100644 index b54d75d1c..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/index.ts +++ /dev/null @@ -1,321 +0,0 @@ -/** - * Sandbox Extension - OS-level sandboxing for bash commands - * - * Uses @anthropic-ai/sandbox-runtime to enforce filesystem and network - * restrictions on bash commands at the OS level (sandbox-exec on macOS, - * bubblewrap on Linux). - * - * Note: this example intentionally overrides the built-in `bash` tool to show - * how built-in tools can be replaced. Alternatively, you could sandbox `bash` - * via `tool_call` input mutation without replacing the tool. - * - * Config files (merged, project takes precedence): - * - ~/.pi/agent/extensions/sandbox.json (global) - * - /.pi/sandbox.json (project-local) - * - * Example .pi/sandbox.json: - * ```json - * { - * "enabled": true, - * "network": { - * "allowedDomains": ["github.com", "*.github.com"], - * "deniedDomains": [] - * }, - * "filesystem": { - * "denyRead": ["~/.ssh", "~/.aws"], - * "allowWrite": [".", "/tmp"], - * "denyWrite": [".env"] - * } - * } - * ``` - * - * Usage: - * - `pi -e ./sandbox` - sandbox enabled with default/config settings - * - `pi -e ./sandbox --no-sandbox` - disable sandboxing - * - `/sandbox` - show current sandbox configuration - * - * Setup: - * 1. Copy sandbox/ directory to ~/.pi/agent/extensions/ - * 2. Run `npm install` in ~/.pi/agent/extensions/sandbox/ - * - * Linux also requires: bubblewrap, socat, ripgrep - */ - -import { spawn } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { SandboxManager, type SandboxRuntimeConfig } from "@anthropic-ai/sandbox-runtime"; -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; -import { type BashOperations, CONFIG_DIR_NAME, createBashTool, getAgentDir } from "@earendil-works/pi-coding-agent"; - -interface SandboxConfig extends SandboxRuntimeConfig { - enabled?: boolean; -} - -const DEFAULT_CONFIG: SandboxConfig = { - enabled: true, - network: { - allowedDomains: [ - "npmjs.org", - "*.npmjs.org", - "registry.npmjs.org", - "registry.yarnpkg.com", - "pypi.org", - "*.pypi.org", - "github.com", - "*.github.com", - "api.github.com", - "raw.githubusercontent.com", - ], - deniedDomains: [], - }, - filesystem: { - denyRead: ["~/.ssh", "~/.aws", "~/.gnupg"], - allowWrite: [".", "/tmp"], - denyWrite: [".env", ".env.*", "*.pem", "*.key"], - }, -}; - -function loadConfig(cwd: string): SandboxConfig { - const projectConfigPath = join(cwd, CONFIG_DIR_NAME, "sandbox.json"); - const globalConfigPath = join(getAgentDir(), "extensions", "sandbox.json"); - - let globalConfig: Partial = {}; - let projectConfig: Partial = {}; - - if (existsSync(globalConfigPath)) { - try { - globalConfig = JSON.parse(readFileSync(globalConfigPath, "utf-8")); - } catch (e) { - console.error(`Warning: Could not parse ${globalConfigPath}: ${e}`); - } - } - - if (existsSync(projectConfigPath)) { - try { - projectConfig = JSON.parse(readFileSync(projectConfigPath, "utf-8")); - } catch (e) { - console.error(`Warning: Could not parse ${projectConfigPath}: ${e}`); - } - } - - return deepMerge(deepMerge(DEFAULT_CONFIG, globalConfig), projectConfig); -} - -function deepMerge(base: SandboxConfig, overrides: Partial): SandboxConfig { - const result: SandboxConfig = { ...base }; - - if (overrides.enabled !== undefined) result.enabled = overrides.enabled; - if (overrides.network) { - result.network = { ...base.network, ...overrides.network }; - } - if (overrides.filesystem) { - result.filesystem = { ...base.filesystem, ...overrides.filesystem }; - } - - const extOverrides = overrides as { - ignoreViolations?: Record; - enableWeakerNestedSandbox?: boolean; - }; - const extResult = result as { ignoreViolations?: Record; enableWeakerNestedSandbox?: boolean }; - - if (extOverrides.ignoreViolations) { - extResult.ignoreViolations = extOverrides.ignoreViolations; - } - if (extOverrides.enableWeakerNestedSandbox !== undefined) { - extResult.enableWeakerNestedSandbox = extOverrides.enableWeakerNestedSandbox; - } - - return result; -} - -function createSandboxedBashOps(): BashOperations { - return { - async exec(command, cwd, { onData, signal, timeout }) { - if (!existsSync(cwd)) { - throw new Error(`Working directory does not exist: ${cwd}`); - } - - const wrappedCommand = await SandboxManager.wrapWithSandbox(command); - - return new Promise((resolve, reject) => { - const child = spawn("bash", ["-c", wrappedCommand], { - cwd, - detached: true, - stdio: ["ignore", "pipe", "pipe"], - }); - - let timedOut = false; - let timeoutHandle: NodeJS.Timeout | undefined; - - if (timeout !== undefined && timeout > 0) { - timeoutHandle = setTimeout(() => { - timedOut = true; - if (child.pid) { - try { - process.kill(-child.pid, "SIGKILL"); - } catch { - child.kill("SIGKILL"); - } - } - }, timeout * 1000); - } - - child.stdout?.on("data", onData); - child.stderr?.on("data", onData); - - child.on("error", (err) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - reject(err); - }); - - const onAbort = () => { - if (child.pid) { - try { - process.kill(-child.pid, "SIGKILL"); - } catch { - child.kill("SIGKILL"); - } - } - }; - - signal?.addEventListener("abort", onAbort, { once: true }); - - child.on("close", (code) => { - if (timeoutHandle) clearTimeout(timeoutHandle); - signal?.removeEventListener("abort", onAbort); - - if (signal?.aborted) { - reject(new Error("aborted")); - } else if (timedOut) { - reject(new Error(`timeout:${timeout}`)); - } else { - resolve({ exitCode: code }); - } - }); - }); - }, - }; -} - -export default function (pi: ExtensionAPI) { - pi.registerFlag("no-sandbox", { - description: "Disable OS-level sandboxing for bash commands", - type: "boolean", - default: false, - }); - - const localCwd = process.cwd(); - const localBash = createBashTool(localCwd); - - let sandboxEnabled = false; - let sandboxInitialized = false; - - pi.registerTool({ - ...localBash, - label: "bash (sandboxed)", - async execute(id, params, signal, onUpdate, _ctx) { - if (!sandboxEnabled || !sandboxInitialized) { - return localBash.execute(id, params, signal, onUpdate); - } - - const sandboxedBash = createBashTool(localCwd, { - operations: createSandboxedBashOps(), - }); - return sandboxedBash.execute(id, params, signal, onUpdate); - }, - }); - - pi.on("user_bash", () => { - if (!sandboxEnabled || !sandboxInitialized) return; - return { operations: createSandboxedBashOps() }; - }); - - pi.on("session_start", async (_event, ctx) => { - const noSandbox = pi.getFlag("no-sandbox") as boolean; - - if (noSandbox) { - sandboxEnabled = false; - ctx.ui.notify("Sandbox disabled via --no-sandbox", "warning"); - return; - } - - const config = loadConfig(ctx.cwd); - - if (!config.enabled) { - sandboxEnabled = false; - ctx.ui.notify("Sandbox disabled via config", "info"); - return; - } - - const platform = process.platform; - if (platform !== "darwin" && platform !== "linux") { - sandboxEnabled = false; - ctx.ui.notify(`Sandbox not supported on ${platform}`, "warning"); - return; - } - - try { - const configExt = config as unknown as { - ignoreViolations?: Record; - enableWeakerNestedSandbox?: boolean; - }; - - await SandboxManager.initialize({ - network: config.network, - filesystem: config.filesystem, - ignoreViolations: configExt.ignoreViolations, - enableWeakerNestedSandbox: configExt.enableWeakerNestedSandbox, - }); - - sandboxEnabled = true; - sandboxInitialized = true; - - const networkCount = config.network?.allowedDomains?.length ?? 0; - const writeCount = config.filesystem?.allowWrite?.length ?? 0; - ctx.ui.setStatus( - "sandbox", - ctx.ui.theme.fg("accent", `🔒 Sandbox: ${networkCount} domains, ${writeCount} write paths`), - ); - ctx.ui.notify("Sandbox initialized", "info"); - } catch (err) { - sandboxEnabled = false; - ctx.ui.notify(`Sandbox initialization failed: ${err instanceof Error ? err.message : err}`, "error"); - } - }); - - pi.on("session_shutdown", async () => { - if (sandboxInitialized) { - try { - await SandboxManager.reset(); - } catch { - // Ignore cleanup errors - } - } - }); - - pi.registerCommand("sandbox", { - description: "Show sandbox configuration", - handler: async (_args, ctx) => { - if (!sandboxEnabled) { - ctx.ui.notify("Sandbox is disabled", "info"); - return; - } - - const config = loadConfig(ctx.cwd); - const lines = [ - "Sandbox Configuration:", - "", - "Network:", - ` Allowed: ${config.network?.allowedDomains?.join(", ") || "(none)"}`, - ` Denied: ${config.network?.deniedDomains?.join(", ") || "(none)"}`, - "", - "Filesystem:", - ` Deny Read: ${config.filesystem?.denyRead?.join(", ") || "(none)"}`, - ` Allow Write: ${config.filesystem?.allowWrite?.join(", ") || "(none)"}`, - ` Deny Write: ${config.filesystem?.denyWrite?.join(", ") || "(none)"}`, - ]; - ctx.ui.notify(lines.join("\n"), "info"); - }, - }); -} diff --git a/packages/coding-agent/examples/extensions/sandbox/package-lock.json b/packages/coding-agent/examples/extensions/sandbox/package-lock.json deleted file mode 100644 index 90ef1c2f5..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/package-lock.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "name": "pi-extension-sandbox", - "version": "1.4.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "pi-extension-sandbox", - "version": "1.4.0", - "dependencies": { - "@anthropic-ai/sandbox-runtime": "^0.0.26" - } - }, - "node_modules/@anthropic-ai/sandbox-runtime": { - "version": "0.0.26", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sandbox-runtime/-/sandbox-runtime-0.0.26.tgz", - "integrity": "sha512-DYV5LSsVMnzq0lbfaYMSpxZPUMAx4+hy343dRss+pVCLIfF62qOhxpYfZ5TmOk1GTDQm5f9wPprMNSStmnsV4w==", - "license": "Apache-2.0", - "dependencies": { - "@pondwader/socks5-server": "^1.0.10", - "@types/lodash-es": "^4.17.12", - "commander": "^12.1.0", - "lodash-es": "^4.17.21", - "shell-quote": "^1.8.3", - "zod": "^3.24.1" - }, - "bin": { - "srt": "dist/cli.js" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@pondwader/socks5-server": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/@pondwader/socks5-server/-/socks5-server-1.0.10.tgz", - "integrity": "sha512-bQY06wzzR8D2+vVCUoBsr5QS2U6UgPUQRmErNwtsuI6vLcyRKkafjkr3KxbtGFf9aBBIV2mcvlsKD1UYaIV+sg==", - "license": "MIT" - }, - "node_modules/@types/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-RDvF6wTulMPjrNdCoYRC8gNR880JNGT8uB+REUpC2Ns4pRqQJhGz90wh7rgdXDPpCczF3VGktDuFGVnz8zP7HA==", - "license": "MIT" - }, - "node_modules/@types/lodash-es": { - "version": "4.17.12", - "resolved": "https://registry.npmjs.org/@types/lodash-es/-/lodash-es-4.17.12.tgz", - "integrity": "sha512-0NgftHUcV4v34VhXm8QBSftKVXtbkBG3ViCjs6+eJ5a6y6Mi/jiFGPc1sC7QK+9BFhWrURE3EOggmWaSxL9OzQ==", - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, - "node_modules/commander": { - "version": "12.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", - "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/lodash-es": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.18.1.tgz", - "integrity": "sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==", - "license": "MIT" - }, - "node_modules/shell-quote": { - "version": "1.8.3", - "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.3.tgz", - "integrity": "sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/zod": { - "version": "3.25.76", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", - "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - } - } -} diff --git a/packages/coding-agent/examples/extensions/sandbox/package.json b/packages/coding-agent/examples/extensions/sandbox/package.json deleted file mode 100644 index 8059d577d..000000000 --- a/packages/coding-agent/examples/extensions/sandbox/package.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "name": "pi-extension-sandbox", - "private": true, - "version": "1.4.0", - "type": "module", - "scripts": { - "clean": "echo 'nothing to clean'", - "build": "echo 'nothing to build'", - "check": "echo 'nothing to check'" - }, - "pi": { - "extensions": [ - "./index.ts" - ] - }, - "dependencies": { - "@anthropic-ai/sandbox-runtime": "0.0.26" - } -} diff --git a/packages/coding-agent/examples/extensions/with-deps/package-lock.json b/packages/coding-agent/examples/extensions/with-deps/package-lock.json index 90c4e6339..2d84832c4 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package-lock.json +++ b/packages/coding-agent/examples/extensions/with-deps/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-extension-with-deps", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { "ms": "^2.1.3" }, diff --git a/packages/coding-agent/examples/extensions/with-deps/package.json b/packages/coding-agent/examples/extensions/with-deps/package.json index f22d436c7..c71ad77f2 100644 --- a/packages/coding-agent/examples/extensions/with-deps/package.json +++ b/packages/coding-agent/examples/extensions/with-deps/package.json @@ -1,7 +1,7 @@ { "name": "pi-extension-with-deps", "private": true, - "version": "1.4.0", + "version": "1.5.0", "type": "module", "scripts": { "clean": "echo 'nothing to clean'", diff --git a/packages/coding-agent/examples/tsconfig.json b/packages/coding-agent/examples/tsconfig.json new file mode 100644 index 000000000..03e1a36e1 --- /dev/null +++ b/packages/coding-agent/examples/tsconfig.json @@ -0,0 +1,180 @@ +// Type-checks the example extensions against the modules extensions see at run time (core/extensions/virtual-modules.ts): +// "@earendil-works/pi-ai" is the extension view, which adds TypeBox's Type, Static and TSchema to the compat entry (the +// view imports "@earendil-works/pi-ai/compat", so that path stays the package's own). +// The other paths repeat the root tsconfig.json; keep them in sync. +// Also the test that imports the subagent example: it type-checks only against the extension view. +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "paths": { + "@earendil-works/chord": [ + "../../../packages/chord/src/index.ts" + ], + "@earendil-works/chord/context": [ + "../../../packages/chord/src/context/index.ts" + ], + "@earendil-works/chord/delta": [ + "../../../packages/chord/src/delta/index.ts" + ], + "@earendil-works/chord/next": [ + "../../../packages/chord/src/next/index.ts" + ], + "@earendil-works/chord/next/plugins": [ + "../../../packages/chord/src/next-node/plugins/index.ts" + ], + "@earendil-works/chord/next/plugins/runtime": [ + "../../../packages/chord/src/next-node/plugins/runtime.ts" + ], + "@test/contracts/*": [ + "../../../packages/chord/test/next-node/fixtures/plugins/contracts/*.ts" + ], + "@example/contracts/*": [ + "../../../packages/chord/test/next/examples/35-plugins/plugins/contracts/*.ts" + ], + "@example/weather/*": [ + "../../../packages/chord/test/next/examples/36-plugin-failures/plugins/weather/*.ts" + ], + "@example/prices/*": [ + "../../../packages/chord/test/next/examples/36-plugin-failures/plugins/prices/*.ts" + ], + "@example/shapes/*": [ + "../../../packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/*.ts" + ], + "@example/shared/*": [ + "../../../packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/*.ts" + ], + "@earendil-works/chord/next/http": [ + "../../../packages/chord/src/next-node/http/index.ts" + ], + "@earendil-works/chord/next/workers": [ + "../../../packages/chord/src/next-node/workers/index.ts" + ], + "@earendil-works/chord/node": [ + "../../../packages/chord/src/node.ts" + ], + "@earendil-works/chord/output": [ + "../../../packages/chord/src/output.ts" + ], + "@earendil-works/chord/telemetry-sqlite": [ + "../../../packages/chord/src/telemetry-sqlite/index.ts" + ], + "@earendil-works/chord/sqlite": [ + "../../../packages/chord/src/sqlite/index.ts" + ], + "@earendil-works/chord/sqlite/node": [ + "../../../packages/chord/src/sqlite/node.ts" + ], + "@earendil-works/chord/sqlite/cloudflare": [ + "../../../packages/chord/src/sqlite/cloudflare.ts" + ], + "@earendil-works/pi-telemetry": [ + "../../../packages/telemetry/src/index.ts" + ], + "@earendil-works/pi-telemetry/testing": [ + "../../../packages/telemetry/src/testing/index.ts" + ], + "@earendil-works/pi-telemetry/render": [ + "../../../packages/telemetry/src/render.ts" + ], + "@earendil-works/pi-codemode": [ + "../../../packages/codemode/src/index.ts" + ], + "@earendil-works/pi-codemode/cloudflare": [ + "../../../packages/codemode/src/cloudflare.ts" + ], + "@earendil-works/pi-codemode/declarations": [ + "../../../packages/codemode/src/declarations.ts" + ], + "@earendil-works/pi-codemode/portable": [ + "../../../packages/codemode/src/portable.ts" + ], + "@earendil-works/pi-codemode/source": [ + "../../../packages/codemode/src/source.ts" + ], + "@earendil-works/pi-codemode/worker": [ + "../../../packages/codemode/src/runtime/worker.ts" + ], + "@earendil-works/pi-schema": [ + "../../../packages/schema/src/index.ts" + ], + "@earendil-works/pi-schema/formats": [ + "../../../packages/schema/src/formats/index.ts" + ], + "@earendil-works/pi-mcp": [ + "../../../packages/mcp/src/index.ts" + ], + "@earendil-works/pi-mcp/oauth": [ + "../../../packages/mcp/src/oauth/index.ts" + ], + "@earendil-works/pi-mcp/testing": [ + "../../../packages/mcp/src/testing/index.ts" + ], + "@earendil-works/pi-ai": [ + "../../../packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts" + ], + "@earendil-works/pi-ai/oauth": [ + "../../../packages/ai/src/oauth.ts" + ], + "@earendil-works/pi-ai/*": [ + "../../../packages/ai/src/*.ts", + "../../../packages/ai/src/providers/*.ts" + ], + "@earendil-works/pi-ai/dist/*": [ + "../../../packages/ai/src/*" + ], + "@earendil-works/pi-durable": [ + "../../../packages/durable/src/index.ts" + ], + "@earendil-works/pi-durable/testing": [ + "../../../packages/durable/src/testing/index.ts" + ], + "@earendil-works/pi-durable/*": [ + "../../../packages/durable/src/*.ts", + "../../../packages/durable/src/*/index.ts", + "../../../packages/durable/src/*" + ], + "@earendil-works/pi-env": [ + "../../../packages/env/src/index.ts" + ], + "@earendil-works/pi-agent-core": [ + "../../../packages/agent/src/index.ts" + ], + "@earendil-works/pi-coding-agent": [ + "../../../packages/coding-agent/src/index.ts" + ], + "@earendil-works/pi-coding-agent/hooks": [ + "../../../packages/coding-agent/src/core/hooks/index.ts" + ], + "@earendil-works/pi-coding-agent/*": [ + "../../../packages/coding-agent/src/*" + ], + "typebox": [ + "../../../node_modules/typebox" + ], + "@earendil-works/pi-tui": [ + "../../../packages/tui/src/index.ts" + ], + "@earendil-works/pi-tui/*": [ + "../../../packages/tui/src/*" + ], + "@earendil-works/pi-agent-old": [ + "../../../packages/agent-old/src/index.ts" + ], + "@earendil-works/pi-agent-old/*": [ + "../../../packages/agent-old/src/*" + ], + "@mariozechner/pi-ai": [ + "../../../packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts" + ] + } + }, + "include": [ + "./extensions/**/*", + "../../*/src/**/*.d.ts", + "../test/suite/regressions/8261-subagent-project-trust.test.ts" + ], + "exclude": [ + "**/node_modules/**", + "./extensions/gondolin/**" + ] +} diff --git a/packages/coding-agent/install-lock/package-lock.json b/packages/coding-agent/install-lock/package-lock.json index 25e273cf6..e37a4fb76 100644 --- a/packages/coding-agent/install-lock/package-lock.json +++ b/packages/coding-agent/install-lock/package-lock.json @@ -1,14 +1,14 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "dependencies": { - "@earendil-works/pi-coding-agent": "1.4.0" + "@earendil-works/pi-coding-agent": "1.5.0" }, "engines": { "node": ">=22.19.0" @@ -386,32 +386,32 @@ } }, "node_modules/@earendil-works/pi-agent-core": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-agent-core/-/pi-agent-core-1.5.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-ai": "1.4.0", - "typebox": "1.3.27" + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-schema": "1.5.0" }, "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-ai": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.5.0.tgz", "license": "MIT", "dependencies": { "@anthropic-ai/sdk": "0.129.0", "@aws-sdk/client-bedrock-runtime": "3.1127.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@google/genai": "2.21.0", "@smithy/node-http-handler": "4.12.1", "http-proxy-agent": "9.1.0", "https-proxy-agent": "9.1.0", "openai": "7.19.0", - "partial-json": "0.1.7", - "typebox": "1.3.27" + "partial-json": "0.1.7" }, "bin": { "pi-ai": "dist/cli.js" @@ -481,8 +481,8 @@ } }, "node_modules/@earendil-works/pi-codemode": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-codemode/-/pi-codemode-1.5.0.tgz", "license": "MIT", "dependencies": { "quickjs-wasi": "3.6.2" @@ -492,15 +492,15 @@ } }, "node_modules/@earendil-works/pi-coding-agent": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-coding-agent/-/pi-coding-agent-1.5.0.tgz", "license": "MIT", "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -511,7 +511,7 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", @@ -527,8 +527,8 @@ } }, "node_modules/@earendil-works/pi-mcp": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-mcp/-/pi-mcp-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-mcp/-/pi-mcp-1.5.0.tgz", "license": "MIT", "dependencies": { "cross-spawn": "7.0.6" @@ -537,17 +537,25 @@ "node": ">=22.19.0" } }, + "node_modules/@earendil-works/pi-schema": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-schema/-/pi-schema-1.5.0.tgz", + "license": "MIT", + "engines": { + "node": ">=22.19.0" + } + }, "node_modules/@earendil-works/pi-telemetry": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.5.0.tgz", "license": "MIT", "engines": { "node": ">=22.19.0" } }, "node_modules/@earendil-works/pi-tui": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-1.4.0.tgz", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@earendil-works/pi-tui/-/pi-tui-1.5.0.tgz", "license": "MIT", "dependencies": { "get-east-asian-width": "1.6.0", @@ -582,6 +590,12 @@ }, "hasInstallScript": true }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.3.tgz", + "integrity": "sha512-SI7kyKHq4XXDRozw+r9GlRZkia1ZkFrpLW98r2IC2Bpr8YkFGE3fvjzmxCG0x78A++69HmDNrDvSMYBZmHGpKA==", + "license": "BlueOak-1.0.0" + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1174,12 +1188,13 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.3.tgz", + "integrity": "sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==", "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.3", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/packages/coding-agent/install-lock/package.json b/packages/coding-agent/install-lock/package.json index 004cf3c90..a8746dfab 100644 --- a/packages/coding-agent/install-lock/package.json +++ b/packages/coding-agent/install-lock/package.json @@ -1,10 +1,10 @@ { "name": "@earendil-works/pi-coding-agent-install", - "version": "1.4.0", + "version": "1.5.0", "private": true, "description": "Lockfile root used by the Pi installer and updater.", "dependencies": { - "@earendil-works/pi-coding-agent": "1.4.0" + "@earendil-works/pi-coding-agent": "1.5.0" }, "overrides": { "protobufjs": "7.6.6" diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 0e452d9b5..705be3d3d 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-coding-agent", - "version": "1.4.0", + "version": "1.5.0", "description": "Coding agent CLI with read, bash, edit, write tools and session management", "type": "module", "piConfig": { @@ -29,23 +29,23 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "npm run build:unbundled && node ../../scripts/build-coding-agent-bundle.mjs", - "build:unbundled": "tsc -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", + "build:unbundled": "tsc -p tsconfig.build.json && node ../../scripts/package-files.mjs executable dist/cli.js dist/rpc-entry.js && npm run copy-assets", "build:binary": "npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../ai run build && npm --prefix ../agent run build && npm run build && bun build --compile --no-compile-autoload-bunfig --no-compile-autoload-dotenv ./dist/bun/cli.js ./src/utils/image-resize-worker.ts ./src/extensions/codemode/worker.ts --outfile dist/pi && npm run copy-binary-assets", - "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/modes/interactive/assets && shx cp src/modes/interactive/assets/*.png dist/modes/interactive/assets/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/", - "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/assets && shx cp src/modes/interactive/assets/*.png dist/assets/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp -r examples dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/", + "copy-assets": "node ../../scripts/package-files.mjs copy \"src/modes/interactive/theme/*.json\" dist/modes/interactive/theme && node ../../scripts/package-files.mjs copy \"src/modes/interactive/assets/*.png\" dist/modes/interactive/assets && node ../../scripts/package-files.mjs copy src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html && node ../../scripts/package-files.mjs copy \"src/core/export-html/vendor/*.js\" dist/core/export-html/vendor", + "copy-binary-assets": "node ../../scripts/package-files.mjs copy package.json README.md CHANGELOG.md docs examples ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist && node ../../scripts/package-files.mjs copy \"src/modes/interactive/theme/*.json\" dist/theme && node ../../scripts/package-files.mjs copy \"src/modes/interactive/assets/*.png\" dist/assets && node ../../scripts/package-files.mjs copy src/core/export-html/template.html dist/export-html && node ../../scripts/package-files.mjs copy \"src/core/export-html/vendor/*.js\" dist/export-html/vendor", "test": "vitest --run", "generate:schemas": "node --import ./src/experimental/source-resolver.ts scripts/generate-schemas.ts", "generate:references": "node --import ./src/experimental/source-resolver.ts scripts/generate-references.ts", "prepublishOnly": "npm run clean && npm run build" }, "dependencies": { - "@earendil-works/pi-agent-core": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-mcp": "1.4.0", - "@earendil-works/pi-tui": "1.4.0", + "@earendil-works/pi-agent-core": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", + "@earendil-works/pi-tui": "1.5.0", "@silvia-odwyer/photon-node": "0.3.4", "brace-expansion": "5.0.12", "chalk": "6.0.0", @@ -56,7 +56,7 @@ "hosted-git-info": "9.0.3", "ignore": "7.0.8", "jiti": "2.7.0", - "minimatch": "10.2.6", + "minimatch": "11.0.3", "proper-lockfile": "4.1.2", "quickjs-wasi": "3.6.2", "semver": "7.8.5", @@ -74,7 +74,6 @@ "@types/proper-lockfile": "4.1.4", "@types/semver": "7.7.1", "marked": "18.0.11", - "shx": "0.3.4", "vitest": "4.1.12" }, "keywords": [ diff --git a/packages/coding-agent/schemas/models.schema.json b/packages/coding-agent/schemas/models.schema.json index bd0619e54..8861f8542 100644 --- a/packages/coding-agent/schemas/models.schema.json +++ b/packages/coding-agent/schemas/models.schema.json @@ -255,13 +255,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "inputTokensAbove", - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "inputTokensAbove": { "type": "number", @@ -283,7 +276,14 @@ "type": "number", "description": "Cache-write cost in USD per million tokens." } - } + }, + "required": [ + "inputTokensAbove", + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "description": "Request-wide pricing tiers. The highest matching input threshold applies to the full request." } @@ -383,12 +383,6 @@ "$defs": { "ModelCost": { "type": "object", - "required": [ - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "input": { "type": "number", @@ -410,13 +404,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "inputTokensAbove", - "input", - "output", - "cacheRead", - "cacheWrite" - ], "properties": { "inputTokensAbove": { "type": "number", @@ -438,11 +425,24 @@ "type": "number", "description": "Cache-write cost in USD per million tokens." } - } + }, + "required": [ + "inputTokensAbove", + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "description": "Request-wide pricing tiers. The highest matching input threshold applies to the full request." } - } + }, + "required": [ + "input", + "output", + "cacheRead", + "cacheWrite" + ] }, "ModelInputLimits": { "type": "object", @@ -628,9 +628,6 @@ }, { "type": "object", - "required": [ - "$var" - ], "properties": { "$var": { "anyOf": [ @@ -651,7 +648,10 @@ "omitWhenOff": { "type": "boolean" } - } + }, + "required": [ + "$var" + ] } ] } @@ -677,9 +677,6 @@ }, { "type": "object", - "required": [ - "$var" - ], "properties": { "$var": { "anyOf": [ @@ -700,7 +697,10 @@ "omitWhenOff": { "type": "boolean" } - } + }, + "required": [ + "$var" + ] } ] } @@ -1089,11 +1089,6 @@ "type": "array", "items": { "type": "object", - "required": [ - "provider", - "model", - "cost" - ], "properties": { "provider": { "type": "string", @@ -1107,6 +1102,11 @@ "$ref": "#/$defs/ModelCost" } }, + "required": [ + "provider", + "model", + "cost" + ], "description": "An Anthropic server-side refusal fallback model with local pricing metadata." }, "maxItems": 3, diff --git a/packages/coding-agent/schemas/settings.schema.json b/packages/coding-agent/schemas/settings.schema.json index 90eaaa3c0..c331ae27a 100644 --- a/packages/coding-agent/schemas/settings.schema.json +++ b/packages/coding-agent/schemas/settings.schema.json @@ -267,6 +267,14 @@ }, "description": "Command used for npm package lookup and installation, in argv form such as [\"mise\", \"exec\", \"node@20\", \"--\", \"npm\"]." }, + "selfUpdateCommand": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "description": "Command `pi update` runs to update Pi itself, in argv form such as [\"mise\", \"use\", \"-g\", \"npm:{packageName}@{version}\"]. Placeholders: {packageName}, {version}, {installSpec}. PI_SELF_UPDATE_COMMAND takes precedence. Global setting only." + }, "collapseChangelog": { "type": "boolean", "description": "Show the condensed changelog after update; use /changelog for the full changelog.", diff --git a/packages/coding-agent/src/cli/args.ts b/packages/coding-agent/src/cli/args.ts index 1cbfc228b..4bcd28dc2 100644 --- a/packages/coding-agent/src/cli/args.ts +++ b/packages/coding-agent/src/cli/args.ts @@ -305,11 +305,11 @@ ${chalk.bold("Options:")} --append-system-prompt Append text or file contents to the system prompt (can be used multiple times) --mode Output mode: text (default), json, or rpc --print, -p Non-interactive mode: process prompt and exit - --continue, -c Continue previous session - --resume, -r Select a session to resume - --session Use specific session file or partial UUID - --session-id Use exact project session ID, creating it if missing - --fork Fork specific session file or partial UUID into a new session + --continue, -c Continue the most recent session for the current directory + --resume, -r Pick a session to resume + --session Open a session by file path or full/partial ID + --session-id Use the current directory's existing session with the ID, creating it otherwise + --fork Fork a session by file path or full/partial ID into the current directory --session-dir Directory for session storage and lookup --no-session Don't save session (ephemeral) --name, -n Set session display name @@ -463,6 +463,7 @@ ${chalk.bold("Environment Variables:")} ${ENV_SESSION_DIR.padEnd(32)} - Session storage directory (overridden by --session-dir) PI_PACKAGE_DIR - Override package directory (for Nix/Guix store paths) PI_OFFLINE - Disable startup network operations when set to 1/true/yes + PI_SELF_UPDATE_COMMAND - Command pi update runs to update pi itself ({packageName}, {version}, {installSpec}) PI_TELEMETRY - Override install telemetry when set to 1/true/yes or 0/false/no PI_SHARE_VIEWER_URL - Base URL for /share command (default: https://pi.dev/session/) diff --git a/packages/coding-agent/src/config.ts b/packages/coding-agent/src/config.ts index 61dd63f38..5a9552d5d 100644 --- a/packages/coding-agent/src/config.ts +++ b/packages/coding-agent/src/config.ts @@ -334,6 +334,32 @@ export function getSelfUpdateCommand( return command; } +/** + * The self-update command configured by the user: `PI_SELF_UPDATE_COMMAND` (whitespace-separated + * argv, no quoting), else the global `selfUpdateCommand` setting. Overrides install-method detection. + */ +export function getConfiguredSelfUpdateArgv(setting: string[] | undefined): string[] | undefined { + const fromEnv = process.env.PI_SELF_UPDATE_COMMAND?.trim().split(/\s+/).filter(Boolean); + if (fromEnv?.length) return fromEnv; + return setting?.length ? setting : undefined; +} + +/** Fill the `{packageName}`, `{version}`, and `{installSpec}` placeholders of a configured self-update command. */ +export function makeConfiguredSelfUpdateCommand( + argv: string[], + release: { packageName: string; version: string }, +): SelfUpdateCommand { + const values: Record = { + packageName: release.packageName, + version: release.version, + installSpec: `${release.packageName}@${release.version}`, + }; + const [command = "", ...args] = argv.map((arg) => + arg.replace(/\{(packageName|version|installSpec)\}/g, (_, name: string) => values[name]), + ); + return makeSelfUpdateCommandStep(command, args); +} + export function getSelfUpdateUnavailableInstruction( packageName: string, npmCommand?: string[], @@ -345,11 +371,10 @@ export function getSelfUpdateUnavailableInstruction( return `Download from: https://github.com/earendil-works/pi/releases/latest`; } const command = getSelfUpdateCommandForMethod(method, packageName, target, npmCommand); - if (command) { - if (isManagedByGlobalPackageManager(method, packageName, npmCommand) && !isSelfUpdatePathWritable()) { - return `This installation is managed by a global ${method} install, but the install path is not writable. Update it yourself with: ${command.display}`; - } - return `This installation is not managed by a global ${method} install. Update it with the package manager, wrapper, or source checkout that provides it.`; + // Path-based detection also matches installs that only live under a package manager's directory, + // such as a wrapper under PNPM_HOME, so name the package manager only when it owns the install. + if (command && isManagedByGlobalPackageManager(method, packageName, npmCommand) && !isSelfUpdatePathWritable()) { + return `This installation is managed by a global ${method} install, but the install path is not writable. Update it yourself with: ${command.display}`; } return `Update ${target.installSpec} using the package manager, wrapper, or source checkout that provides this installation.`; } diff --git a/packages/coding-agent/src/core/agent-session-runtime.ts b/packages/coding-agent/src/core/agent-session-runtime.ts index c1d7655dc..81c4264cf 100644 --- a/packages/coding-agent/src/core/agent-session-runtime.ts +++ b/packages/coding-agent/src/core/agent-session-runtime.ts @@ -1,6 +1,6 @@ import { constants, copyFileSync, existsSync, mkdirSync } from "node:fs"; -import { basename, join, parse, resolve } from "node:path"; -import { resolvePath } from "../utils/paths.ts"; +import { basename, join, parse } from "node:path"; +import { pathsEqual, resolvePath } from "../utils/paths.ts"; import type { AgentSession } from "./agent-session.ts"; import type { AgentSessionRuntimeDiagnostic, AgentSessionServices } from "./agent-session-services.ts"; import type { @@ -368,7 +368,7 @@ export class AgentSessionRuntime { } let destinationPath = join(sessionDir, basename(resolvedPath)); - const sourceAlreadyStored = resolve(destinationPath) === resolvedPath; + const sourceAlreadyStored = pathsEqual(destinationPath, resolvedPath); if (!sourceAlreadyStored) { const { name, ext } = parse(destinationPath); let suffix = 1; diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index c1529f019..dbe36c365 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -29,6 +29,7 @@ import { type BeforeToolCallResult, type PrepareNextTurnContext, runToolCall, + type StreamFn, type ThinkingLevel, } from "@earendil-works/pi-agent-core"; import { contentText, getCurrentSystemMessage, getCurrentTools, retryDelayMs } from "@earendil-works/pi-ai"; @@ -158,6 +159,18 @@ import { type VirtualModelStateData, } from "./virtual-models.ts"; +/** + * Stream functions that resolve auth through a given ModelRuntime themselves, such as the one + * createAgentSession installs. Summaries sent through them must not pass pre-resolved credentials: + * the runtime would take them as an explicit API key instead of the stored OAuth login (#11520). + */ +const runtimeStreamFunctions = new WeakMap(); + +/** Mark `streamFn` as resolving auth through `modelRuntime` on its own. */ +export function markRuntimeStreamFunction(streamFn: StreamFn, modelRuntime: ModelRuntime): void { + runtimeStreamFunctions.set(streamFn, modelRuntime); +} + // ============================================================================ // Skill Block Parsing // ============================================================================ @@ -590,6 +603,9 @@ export class AgentSession { if (this.agent.streamFunction === streamSimple) { return { ...(await this._getRequiredRequestAuth(model, signal)), thinkingLevel }; } + if (runtimeStreamFunctions.get(this.agent.streamFunction) === this._modelRuntime) { + return { model, thinkingLevel }; + } try { const result = await this._modelRuntime.getAuth(model, { signal }); diff --git a/packages/coding-agent/src/core/compaction/branch-summarization.ts b/packages/coding-agent/src/core/compaction/branch-summarization.ts index 69bd75133..6f8bc0cb3 100644 --- a/packages/coding-agent/src/core/compaction/branch-summarization.ts +++ b/packages/coding-agent/src/core/compaction/branch-summarization.ts @@ -15,7 +15,12 @@ import { createCompactionSummaryMessage, createCustomMessage, } from "../messages.ts"; -import type { ReadonlySessionManager, SessionEntry } from "../session-manager.ts"; +import { + type ContextEditEntry, + projectContextEntry, + type ReadonlySessionManager, + type SessionEntry, +} from "../session-manager.ts"; import { completeSummarization, estimateTokens, getSummarizationFailure } from "./compaction.ts"; import { computeFileLists, @@ -155,14 +160,15 @@ export function collectEntriesForBranchSummary( * Extract AgentMessage from a session entry. * Similar to getMessageFromEntry in compaction.ts but also handles compaction entries. */ -function getMessageFromEntry(entry: SessionEntry): AgentMessage | undefined { +function getMessageFromEntry(entry: SessionEntry, edit?: ContextEditEntry): AgentMessage | undefined { switch (entry.type) { case "message": // Skip tool results - context is in assistant's tool call if (entry.message.role === "toolResult") return undefined; - return entry.message; + return edit ? projectContextEntry(entry, edit)[0] : entry.message; case "custom_message": + if (edit) return projectContextEntry(entry, edit)[0]; return createCustomMessage(entry.customType, entry.content, entry.display, entry.details, entry.timestamp); case "branch_summary": @@ -217,10 +223,16 @@ export function prepareBranchEntries(entries: SessionEntry[], tokenBudget: numbe } } + // Context edits hide or replace earlier entries in model context; honor the latest edit per target. + const edits = new Map(); + for (const entry of entries) { + if (entry.type === "context_edit") edits.set(entry.targetId, entry); + } + // Second pass: walk from newest to oldest, adding messages until token budget for (let i = entries.length - 1; i >= 0; i--) { const entry = entries[i]; - const message = getMessageFromEntry(entry); + const message = getMessageFromEntry(entry, edits.get(entry.id)); if (!message) continue; // Extract file ops from assistant messages (tool calls) diff --git a/packages/coding-agent/src/core/exec.ts b/packages/coding-agent/src/core/exec.ts index cf4205f7f..b6e845f45 100644 --- a/packages/coding-agent/src/core/exec.ts +++ b/packages/coding-agent/src/core/exec.ts @@ -2,10 +2,14 @@ * Shared command execution utilities for extensions and custom tools. */ +import { constants as bufferConstants } from "node:buffer"; import { spawn } from "node:child_process"; import { StringDecoder } from "node:string_decoder"; import { waitForChildProcess } from "../utils/child-process.ts"; +const DEFAULT_EXEC_MAX_BUFFER = Math.min(256 * 1024 * 1024, bufferConstants.MAX_STRING_LENGTH); +const FORCE_KILL_DELAY_MS = 5000; + /** * Options for executing shell commands. */ @@ -16,6 +20,11 @@ export interface ExecOptions { timeout?: number; /** Working directory */ cwd?: string; + /** + * Combined stdout/stderr byte limit; exceeding it terminates the command with `truncated: true`. + * Nonnegative integer up to the runtime string limit. Default: 256 MiB, capped by that limit. + */ + maxBuffer?: number; } /** @@ -25,12 +34,15 @@ export interface ExecResult { stdout: string; stderr: string; code: number; + /** The command was terminated by pi: timeout, abort, or output over `maxBuffer`. */ killed: boolean; + /** Output exceeded `maxBuffer`; stdout and stderr are incomplete and `code` is nonzero. */ + truncated: boolean; } /** * Execute a shell command and return stdout/stderr/code. - * Supports timeout and abort signal. + * Supports timeout, abort signal, and an output cap (`maxBuffer`). */ export async function execCommand( command: string, @@ -38,79 +50,109 @@ export async function execCommand( cwd: string, options?: ExecOptions, ): Promise { - return new Promise((resolve) => { - const proc = spawn(command, args, { - cwd, - shell: false, - stdio: ["ignore", "pipe", "pipe"], - }); + const maxBuffer = options?.maxBuffer ?? DEFAULT_EXEC_MAX_BUFFER; + if (!Number.isInteger(maxBuffer) || maxBuffer < 0 || maxBuffer > bufferConstants.MAX_STRING_LENGTH) { + throw new RangeError(`maxBuffer must be an integer between 0 and ${bufferConstants.MAX_STRING_LENGTH} bytes`); + } - let stdout = ""; - let stderr = ""; - // Decode across chunks: a multi-byte UTF-8 character can be split between two chunks, - // and decoding each chunk separately would corrupt it (earendil-works/pi#11312). - const stdoutDecoder = new StringDecoder("utf8"); - const stderrDecoder = new StringDecoder("utf8"); - let killed = false; - let timeoutId: NodeJS.Timeout | undefined; - - const killProcess = () => { - if (!killed) { - killed = true; - proc.kill("SIGTERM"); - // Force kill after 5 seconds if SIGTERM doesn't work - setTimeout(() => { - if (!proc.killed) { - proc.kill("SIGKILL"); - } - }, 5000); - } - }; - - // Handle abort signal - if (options?.signal) { - if (options.signal.aborted) { - killProcess(); - } else { - options.signal.addEventListener("abort", killProcess, { once: true }); - } - } - - // Handle timeout - if (options?.timeout && options.timeout > 0) { - timeoutId = setTimeout(() => { - killProcess(); - }, options.timeout); - } - - proc.stdout?.on("data", (data) => { - stdout += stdoutDecoder.write(data); - }); - - proc.stderr?.on("data", (data) => { - stderr += stderrDecoder.write(data); - }); - - // Wait for process termination without hanging on inherited stdio handles - // held open by detached descendants. - waitForChildProcess(proc) - .then((code) => { - if (timeoutId) clearTimeout(timeoutId); - stdout += stdoutDecoder.end(); - stderr += stderrDecoder.end(); - if (options?.signal) { - options.signal.removeEventListener("abort", killProcess); - } - resolve({ stdout, stderr, code: code ?? 0, killed }); - }) - .catch((error: Error) => { - if (timeoutId) clearTimeout(timeoutId); - stdout += stdoutDecoder.end(); - stderr += stderrDecoder.end(); - if (options?.signal) { - options.signal.removeEventListener("abort", killProcess); - } - resolve({ stdout, stderr: stderr || error.message, code: 1, killed }); - }); + const proc = spawn(command, args, { + cwd, + shell: false, + stdio: ["ignore", "pipe", "pipe"], }); + + let stdout = ""; + let stderr = ""; + // Decode across chunks: a multi-byte UTF-8 character can be split between two chunks, + // and decoding each chunk separately would corrupt it (earendil-works/pi#11312). + const stdoutDecoder = new StringDecoder("utf8"); + const stderrDecoder = new StringDecoder("utf8"); + let outputBytes = 0; + let truncated = false; + let killed = false; + let timeoutId: NodeJS.Timeout | undefined; + let forceKillId: NodeJS.Timeout | undefined; + + const killProcess = () => { + if (killed) return; + killed = true; + proc.kill("SIGTERM"); + forceKillId = setTimeout(() => { + // `proc.killed` only says a signal was sent, so check whether the child actually exited. + if (proc.exitCode === null && proc.signalCode === null) { + proc.kill("SIGKILL"); + } + }, FORCE_KILL_DELAY_MS); + }; + + if (options?.signal) { + if (options.signal.aborted) { + killProcess(); + } else { + options.signal.addEventListener("abort", killProcess, { once: true }); + } + } + + if (options?.timeout && options.timeout > 0) { + timeoutId = setTimeout(killProcess, options.timeout); + } + + // Keep only bytes within the combined cap (#11360). Close both pipes on overflow so writing + // descendants cannot keep waitForChildProcess busy after the child exits. Pausing is not + // enough: Node resumes the child's stdio on exit. + const takeWithinCap = (data: Buffer): Buffer | undefined => { + if (truncated) return undefined; + const remaining = maxBuffer - outputBytes; + if (data.length <= remaining) { + outputBytes += data.length; + return data; + } + truncated = true; + proc.stdout?.destroy(); + proc.stderr?.destroy(); + killProcess(); + return remaining > 0 ? data.subarray(0, remaining) : undefined; + }; + + proc.stdout?.on("data", (data: Buffer) => { + const chunk = takeWithinCap(data); + if (chunk) stdout += stdoutDecoder.write(chunk); + }); + + proc.stderr?.on("data", (data: Buffer) => { + const chunk = takeWithinCap(data); + if (chunk) stderr += stderrDecoder.write(chunk); + }); + + let code: number | null; + try { + // Do not hang on inherited stdio handles held open by detached descendants. + code = await waitForChildProcess(proc); + } catch (error) { + if (!(error instanceof Error)) throw error; + // Child-process errors (e.g. ENOENT) are failed executions, not host failures. + code = 1; + if (!stderr) stderr = error.message; + } finally { + clearTimeout(timeoutId); + clearTimeout(forceKillId); + options?.signal?.removeEventListener("abort", killProcess); + } + + // On truncation, discard any incomplete UTF-8 suffix buffered by either decoder. + // Flushing it would append a replacement character that was not in the accepted output. + if (!truncated) { + stdout += stdoutDecoder.end(); + stderr += stderrDecoder.end(); + } + + const exitCode = code ?? 0; + return { + stdout, + stderr, + // A truncated result must never read as success. + code: truncated && exitCode === 0 ? 1 : exitCode, + killed, + truncated, + }; } diff --git a/packages/coding-agent/src/core/export-html/tool-renderer.ts b/packages/coding-agent/src/core/export-html/tool-renderer.ts index 1f8c1fd63..aac0e199b 100644 --- a/packages/coding-agent/src/core/export-html/tool-renderer.ts +++ b/packages/coding-agent/src/core/export-html/tool-renderer.ts @@ -7,6 +7,7 @@ import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; import type { Component } from "@earendil-works/pi-tui"; +import { keyHint, keyText } from "../../modes/interactive/components/keybinding-hints.ts"; import type { Theme } from "../../modes/interactive/theme/theme.ts"; import type { ToolRenderContext, ToolRenderers } from "../extensions/types.ts"; import { ansiLinesToHtml } from "./ansi-to-html.ts"; @@ -94,6 +95,8 @@ export function createToolHtmlRenderer(deps: ToolHtmlRendererDeps): ToolHtmlRend isError, durationMs: undefined, outputPad: 1, + keyText, + keyHint, }; }; diff --git a/packages/coding-agent/src/core/extensions/index.ts b/packages/coding-agent/src/core/extensions/index.ts index 1465e8267..99c0044cd 100644 --- a/packages/coding-agent/src/core/extensions/index.ts +++ b/packages/coding-agent/src/core/extensions/index.ts @@ -109,6 +109,7 @@ export type { InputEvent, InputEventResult, InputSource, + KeybindingHints, KeybindingsManager, LoadExtensionsResult, LsToolCallEvent, diff --git a/packages/coding-agent/src/core/extensions/loader.ts b/packages/coding-agent/src/core/extensions/loader.ts index cbc1100fc..ff9f127c4 100644 --- a/packages/coding-agent/src/core/extensions/loader.ts +++ b/packages/coding-agent/src/core/extensions/loader.ts @@ -11,7 +11,7 @@ import type { Provider } from "@earendil-works/pi-ai"; import type { KeyId } from "@earendil-works/pi-tui"; import type { createJiti } from "jiti"; import { CONFIG_DIR_NAME, getAgentDir, isBunBinary, isBundledNode } from "../../config.ts"; -import { resolvePath } from "../../utils/paths.ts"; +import { pathsEqual, resolvePath } from "../../utils/paths.ts"; import { createEventBus, type EventBus } from "../event-bus.ts"; import type { ExecOptions } from "../exec.ts"; import { execCommand } from "../exec.ts"; @@ -88,10 +88,11 @@ function getAliases(): Record { const piCodingAgentEntry = packageIndex; const piAgentCoreEntry = resolveWorkspaceOrImport("agent/dist/index.js", "@earendil-works/pi-agent-core"); const piTuiEntry = resolveWorkspaceOrImport("tui/dist/index.js", "@earendil-works/pi-tui"); - // Extensions resolve the pi-ai root to the compat entrypoint (a strict - // superset of the core entrypoint): existing extensions using the old - // global API keep working at runtime until compat is removed. - const piAiCompatEntry = resolveWorkspaceOrImport("ai/dist/compat.js", "@earendil-works/pi-ai/compat"); + // Extensions resolve the pi-ai root to the extension view: the compat + // entrypoint (a strict superset of the core entrypoint) plus TypeBox's Type, + // Static and TSchema, so extensions using the old global API and pi-ai's + // former TypeBox re-exports keep working. + const piAiCompatEntry = path.resolve(__dirname, "pi-ai-extension-view.js"); const piAiModelsEntry = resolveWorkspaceOrImport("ai/dist/models.js", "@earendil-works/pi-ai/models"); const piAiOauthEntry = resolveWorkspaceOrImport("ai/dist/oauth.js", "@earendil-works/pi-ai/oauth"); const piAiProvidersEntry = resolveWorkspaceOrImport( @@ -146,7 +147,7 @@ export function clearExtensionCache(): void { function useExtensionCacheCwd(cwd: string): ExtensionCacheToken { const resolvedCwd = resolvePath(cwd); - if (extensionCacheCwd !== undefined && extensionCacheCwd !== resolvedCwd) { + if (extensionCacheCwd !== undefined && !pathsEqual(extensionCacheCwd, resolvedCwd)) { clearExtensionCache(); } extensionCacheCwd = resolvedCwd; diff --git a/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts b/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts new file mode 100644 index 000000000..e9c6808f7 --- /dev/null +++ b/packages/coding-agent/src/core/extensions/pi-ai-extension-view.ts @@ -0,0 +1,28 @@ +/** + * The `@earendil-works/pi-ai` module extensions import: pi-ai's compat entry point plus TypeBox's `Type`, `Static` and + * `TSchema`, which pi-ai's main entry no longer re-exports, and a `StringEnum` built with TypeBox (`TUnsafe`), so + * extensions that use it inside `Type.Object` type-check and infer as before. The TypeBox is the coding agent's own, + * the same one extensions get for `typebox`. + */ + +import { type TUnsafe, Type } from "typebox"; + +export * from "@earendil-works/pi-ai/compat"; +export type { Static, TSchema } from "typebox"; +export { Type }; + +/** + * Creates a string enum schema compatible with Google's API and other providers that don't support anyOf/const + * patterns: `{ type: "string", enum: [...] }`, typed for TypeBox's `Static`. + */ +export function StringEnum( + values: T, + options?: { description?: string; default?: T[number] }, +): TUnsafe { + return Type.Unsafe({ + type: "string", + enum: values as unknown as T[number][], + ...(options?.description && { description: options.description }), + ...(options?.default && { default: options.default }), + }); +} diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts index 8bb7fbb5e..098f7fb0d 100644 --- a/packages/coding-agent/src/core/extensions/types.ts +++ b/packages/coding-agent/src/core/extensions/types.ts @@ -49,6 +49,7 @@ import type { Component, EditorComponent, EditorTheme, + Keybinding, KeyId, OverlayHandle, OverlayOptions, @@ -143,6 +144,19 @@ export interface WorkingIndicatorOptions { export type AutocompleteProviderFactory = (current: AutocompleteProvider) => AutocompleteProvider; export type EditorFactory = (tui: TUI, theme: EditorTheme, keybindings: KeybindingsManager) => EditorComponent; +/** + * Formats the host's configured keybindings for display. + * Extensions should use these instead of importing `keyText`/`keyHint`: an import may resolve to another copy of + * pi-coding-agent, which does not see the host's keybindings. + */ +export interface KeybindingHints { + /** Configured keys for a keybinding id, for example `"ctrl+o"`. Empty when the id has no keys. */ + keyText(keybinding: Keybinding): string; + + /** Configured keys followed by a description, styled as a hint, for example `"ctrl+o to expand"`. */ + keyHint(keybinding: Keybinding, description: string): string; +} + /** * UI context for extensions to request interactive UI. * Each mode (interactive, RPC, print) provides its own implementation. @@ -470,7 +484,7 @@ export interface ToolRenderResultOptions { } /** Context passed to tool renderers. */ -export interface ToolRenderContext { +export interface ToolRenderContext extends KeybindingHints { /** Current tool call arguments. Shared across call/result renders for the same tool call. */ args: TArgs; /** Unique id for this tool execution. Stable across call/result renders for the same tool call. */ @@ -1538,7 +1552,7 @@ export interface SessionBeforeTreeResult { // Message and Entry Rendering // ============================================================================ -export interface MessageRenderOptions { +export interface MessageRenderOptions extends KeybindingHints { expanded: boolean; /** Horizontal padding configured by the outputPad setting. */ outputPad: number; @@ -1556,7 +1570,7 @@ export interface MarkdownTransformContext { export type MarkdownTransformer = (markdown: string, context: MarkdownTransformContext) => string; -export interface EntryRenderOptions { +export interface EntryRenderOptions extends KeybindingHints { expanded: boolean; } diff --git a/packages/coding-agent/src/core/extensions/virtual-modules.ts b/packages/coding-agent/src/core/extensions/virtual-modules.ts index 4b8449338..3c30317d0 100644 --- a/packages/coding-agent/src/core/extensions/virtual-modules.ts +++ b/packages/coding-agent/src/core/extensions/virtual-modules.ts @@ -1,5 +1,4 @@ import * as bundledPiAgentCore from "@earendil-works/pi-agent-core"; -import * as bundledPiAiCompat from "@earendil-works/pi-ai/compat"; import * as bundledPiAiModels from "@earendil-works/pi-ai/models"; import * as bundledPiAiOauth from "@earendil-works/pi-ai/oauth"; import * as bundledPiAiProviders from "@earendil-works/pi-ai/providers/all"; @@ -10,6 +9,7 @@ import * as bundledTypeboxValue from "typebox/value"; // This import is safe because loader.ts exports are not re-exported from index.ts. // Extensions can therefore import from @earendil-works/pi-coding-agent. import * as bundledPiCodingAgent from "../../index.ts"; +import * as bundledPiAiExtensionView from "./pi-ai-extension-view.ts"; /** Modules available to extensions in source and compiled binary runtimes. */ export const VIRTUAL_MODULES: Record = { @@ -22,18 +22,19 @@ export const VIRTUAL_MODULES: Record = { "@earendil-works/pi-agent-core": bundledPiAgentCore, "@earendil-works/pi-tui": bundledPiTui, // Extensions resolve the pi-ai root to the compat entrypoint (a strict - // superset of the core entrypoint): existing extensions using the old - // global API keep working at runtime until compat is removed. - "@earendil-works/pi-ai": bundledPiAiCompat, - "@earendil-works/pi-ai/compat": bundledPiAiCompat, + // superset of the core entrypoint) plus TypeBox's Type, Static and TSchema: + // existing extensions using the old global API and pi-ai's former TypeBox + // re-exports keep working. + "@earendil-works/pi-ai": bundledPiAiExtensionView, + "@earendil-works/pi-ai/compat": bundledPiAiExtensionView, "@earendil-works/pi-ai/models": bundledPiAiModels, "@earendil-works/pi-ai/oauth": bundledPiAiOauth, "@earendil-works/pi-ai/providers/all": bundledPiAiProviders, "@earendil-works/pi-coding-agent": bundledPiCodingAgent, "@mariozechner/pi-agent-core": bundledPiAgentCore, "@mariozechner/pi-tui": bundledPiTui, - "@mariozechner/pi-ai": bundledPiAiCompat, - "@mariozechner/pi-ai/compat": bundledPiAiCompat, + "@mariozechner/pi-ai": bundledPiAiExtensionView, + "@mariozechner/pi-ai/compat": bundledPiAiExtensionView, "@mariozechner/pi-ai/models": bundledPiAiModels, "@mariozechner/pi-ai/oauth": bundledPiAiOauth, "@mariozechner/pi-ai/providers/all": bundledPiAiProviders, diff --git a/packages/coding-agent/src/core/mcp-servers.ts b/packages/coding-agent/src/core/mcp-servers.ts index 1f6414928..46b4241d5 100644 --- a/packages/coding-agent/src/core/mcp-servers.ts +++ b/packages/coding-agent/src/core/mcp-servers.ts @@ -112,11 +112,13 @@ export interface McpOAuthConfig { */ clientName?: string; /** - * How pi identifies itself without `clientId`. `dcr` (default): dynamic client registration. `cimd`: - * pi's Client ID Metadata Document on pi.dev, for authorization servers that allow pi by that URL. The - * server must support it for public clients, and the callback must use the default path `/callback`. + * How pi identifies itself without `clientId`. `auto` (default): pi's Client ID Metadata Document on + * pi.dev when the authorization server supports it for public clients, otherwise dynamic client + * registration; with `clientName` or a `callbackUrl` other than `/callback` on localhost or 127.0.0.1, + * always dynamic client registration. `cimd`: always the document, and sign-in fails without support. + * `dcr`: always dynamic client registration. */ - clientRegistration?: "dcr" | "cimd"; + clientRegistration?: "auto" | "dcr" | "cimd"; /** * Authorization server metadata document (RFC 8414 or OpenID Connect discovery) to use instead of * discovery through the server, for servers that advertise a wrong authorization server or none. @@ -134,6 +136,16 @@ export function isLoopbackRedirectUri(value: string): boolean { return url.protocol === "http:" && LOOPBACK_HOSTS.includes(url.hostname) && url.search === "" && url.hash === ""; } +/** + * Whether pi's Client ID Metadata Documents list a redirect URI for the loopback `callbackUrl`: they list + * `/callback` on localhost and 127.0.0.1, on any port. Without `callbackUrl`, pi uses such a URI. + */ +export function isClientMetadataDocumentCallback(callbackUrl: string | undefined): boolean { + if (callbackUrl === undefined) return true; + const url = new URL(callbackUrl); + return url.hostname !== "[::1]" && url.pathname === "/callback"; +} + export interface McpHttpServerConfig extends McpServerConfigBase { type?: "http"; url: string; @@ -188,13 +200,13 @@ function validateOAuth(value: unknown): string | undefined { if (value.clientName !== undefined && (typeof value.clientName !== "string" || !value.clientName.trim())) { return "oauth.clientName must be a non-empty string"; } - if (value.clientRegistration !== undefined && value.clientRegistration !== "dcr") { - if (value.clientRegistration !== "cimd") return 'oauth.clientRegistration must be "dcr" or "cimd"'; + const registration = value.clientRegistration; + if (registration !== undefined && registration !== "auto" && registration !== "dcr") { + if (registration !== "cimd") return 'oauth.clientRegistration must be "auto", "dcr", or "cimd"'; if (value.clientId !== undefined || value.clientName !== undefined) { return 'oauth.clientRegistration "cimd" cannot be combined with oauth.clientId or oauth.clientName'; } - const callback = typeof value.callbackUrl === "string" ? new URL(value.callbackUrl) : undefined; - if (callback && (callback.hostname === "[::1]" || callback.pathname !== "/callback")) { + if (!isClientMetadataDocumentCallback(typeof value.callbackUrl === "string" ? value.callbackUrl : undefined)) { return 'oauth.clientRegistration "cimd" requires oauth.callbackUrl on localhost or 127.0.0.1 with path /callback'; } } diff --git a/packages/coding-agent/src/core/model-config.ts b/packages/coding-agent/src/core/model-config.ts index 553ff35ba..200cdeec0 100644 --- a/packages/coding-agent/src/core/model-config.ts +++ b/packages/coding-agent/src/core/model-config.ts @@ -1,20 +1,21 @@ /** Immutable, credential-blind models.json snapshot. */ import { readFile } from "node:fs/promises"; -import { ProviderCompatSchema } from "@earendil-works/pi-ai/providers/compat-schema"; -import { - ModelCostSchema, - ModelInputLimitsSchema, - ModelInputModalitySchema, - ModelPromptCacheSchema, - ThinkingLevelMapSchema, -} from "@earendil-works/pi-ai/providers/model-schema"; import { type Static, Type } from "typebox"; import { Compile } from "typebox/compile"; import type { TLocalizedValidationError } from "typebox/error"; import { stripJsonComments } from "../utils/json.ts"; import { normalizePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; +import { + ModelCostSchema, + ModelInputLimitsSchema, + ModelInputModalitySchema, + ModelPromptCacheSchema, + PartialModelCostSchema, + ProviderCompatSchema, + ThinkingLevelMapSchema, +} from "./pi-ai-schemas.ts"; const SamplingParamsSchema = Type.Record(Type.String(), Type.Unknown()); const SamplingParamsByThinkingLevelSchema = Type.Object( @@ -74,7 +75,7 @@ const ModelOverrideSchema = Type.Object({ thinkingLevelMap: Type.Optional(ThinkingLevelMapSchema), input: Type.Optional(Type.Array(ModelInputModalitySchema)), inputLimits: Type.Optional(ModelInputLimitsSchema), - cost: Type.Optional(Type.Partial(ModelCostSchema)), + cost: Type.Optional(PartialModelCostSchema), promptCache: Type.Optional(ModelPromptCacheSchema), contextWindow: Type.Optional( Type.Number({ exclusiveMinimum: 0, description: ModelFieldDescriptions.contextWindow }), diff --git a/packages/coding-agent/src/core/package-manager.ts b/packages/coding-agent/src/core/package-manager.ts index 3bed70557..5fd93754e 100644 --- a/packages/coding-agent/src/core/package-manager.ts +++ b/packages/coding-agent/src/core/package-manager.ts @@ -43,22 +43,25 @@ import { spawnProcess, spawnProcessSync } from "../utils/child-process.ts"; import { type GitSource, parseGitUrl } from "../utils/git.ts"; import { canonicalizePath, + getPathComparisonKey, isLocalPath, markPathIgnoredByCloudSync, normalizePath, + pathsEqual, resolvePath, } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; import { isStdoutTakenOver } from "./output-guard.ts"; import { type PiManifest, readPiManifest } from "./pi-manifest.ts"; import type { PackageSource, SettingsManager } from "./settings-manager.ts"; -import { BUILTIN_PATH_PREFIX } from "./source-info.ts"; +import { BUILTIN_PATH_PREFIX, getResourcePathKey } from "./source-info.ts"; const NETWORK_TIMEOUT_MS = 10000; const UPDATE_CHECK_CONCURRENCY = 4; const GIT_UPDATE_CONCURRENCY = 4; -function isOfflineModeEnabled(): boolean { +/** Whether PI_OFFLINE is set; package installs and updates then skip all network access. */ +export function isOfflineModeEnabled(): boolean { const value = process.env.PI_OFFLINE; if (!value) return false; return value === "1" || value.toLowerCase() === "true" || value.toLowerCase() === "yes"; @@ -106,7 +109,13 @@ export interface PackageInstallFailure { export type MissingSourceAction = "install" | "skip" | "error"; export interface ProgressEvent { - type: "start" | "progress" | "complete" | "error"; + /** + * `start` is followed by exactly one `complete` or `error` for the same source. + * `progress` carries an intermediate message or warning. + * `result` reports an outcome outside that pairing, e.g. one changed package + * after update(): `npm:foo 1.2.0 -> 1.3.0`. + */ + type: "start" | "progress" | "complete" | "error" | "result"; action: "install" | "remove" | "update" | "clone" | "pull"; source: string; message?: string; @@ -181,12 +190,25 @@ interface ConfiguredUpdateSource { interface NpmUpdateTarget extends ConfiguredUpdateSource { parsed: NpmSource; + /** Version installed before the update, if any. */ + installedVersion?: string; + /** Newer version reported by the registry, if the lookup succeeded. */ + targetVersion?: string; } interface GitUpdateTarget extends ConfiguredUpdateSource { parsed: GitSource; } +/** A configured package whose installed npm version or git commit changed during update(). */ +interface UpdatedPackage extends ConfiguredUpdateSource { + type: "npm" | "git"; + /** npm version or git commit before the update; undefined if it was not installed. */ + from?: string; + /** npm version or git commit after the update. */ + to?: string; +} + interface ResourceAccumulator { extensions: Map; skills: Map; @@ -1188,17 +1210,18 @@ export class DefaultPackageManager implements PackageManager { } } - const npmCheckTasks = npmCandidates.map((entry) => async () => ({ - entry, - shouldUpdate: await this.shouldUpdateNpmSource(entry.parsed, entry.scope), - })); + const npmCheckTasks = npmCandidates.map((entry) => async () => { + const check = await this.shouldUpdateNpmSource(entry.parsed, entry.scope); + return { + entry: { ...entry, installedVersion: check.installedVersion, targetVersion: check.targetVersion }, + shouldUpdate: check.shouldUpdate, + }; + }); const npmCheckResults = await this.runWithConcurrency(npmCheckTasks, UPDATE_CHECK_CONCURRENCY); const userNpmUpdates: NpmUpdateTarget[] = []; const projectNpmUpdates: NpmUpdateTarget[] = []; - for (const result of npmCheckResults) { - if (!result.shouldUpdate) { - continue; - } + const npmUpdates = npmCheckResults.filter((result) => result.shouldUpdate); + for (const result of npmUpdates) { if (result.entry.scope === "user") { userNpmUpdates.push(result.entry); } else { @@ -1213,32 +1236,82 @@ export class DefaultPackageManager implements PackageManager { if (projectNpmUpdates.length > 0) { tasks.push(this.updateNpmBatch(projectNpmUpdates, "project")); } + const gitUpdates: UpdatedPackage[] = []; if (gitCandidates.length > 0) { const gitTasks = gitCandidates.map( (entry) => async () => this.withProgress("update", entry.source, `Updating ${entry.source}...`, async () => { + const targetDir = this.getGitInstallPath(entry.parsed, entry.scope); + const from = await this.getGitHead(targetDir); await this.updateGit(entry.parsed, entry.scope); + const to = await this.getGitHead(targetDir); + if (from !== to) { + gitUpdates.push({ source: entry.source, type: "git", scope: entry.scope, from, to }); + } }), ); tasks.push(this.runWithConcurrency(gitTasks, GIT_UPDATE_CONCURRENCY).then(() => {})); } await Promise.all(tasks); + + const updated: UpdatedPackage[] = []; + for (const { entry } of npmUpdates) { + const installedPath = this.getManagedNpmInstallPath(entry.parsed, entry.scope); + const to = existsSync(installedPath) ? this.getInstalledNpmVersion(installedPath) : undefined; + if (to !== entry.installedVersion) { + updated.push({ source: entry.source, type: "npm", scope: entry.scope, from: entry.installedVersion, to }); + } + } + // Keep git results in configured order, independent of completion order. + for (const entry of gitCandidates) { + const update = gitUpdates.find((u) => u.source === entry.source && u.scope === entry.scope); + if (update) updated.push(update); + } + + // Report each changed package, e.g. "npm:foo 1.2.0 -> 1.3.0". + for (const pkg of updated) { + const format = (value: string | undefined) => + value === undefined ? "(none)" : pkg.type === "git" ? value.slice(0, 7) : value; + const scope = pkg.scope === "project" ? " (project)" : ""; + this.emitProgress({ + type: "result", + action: "update", + source: pkg.source, + message: `${pkg.source}${scope} ${format(pkg.from)} -> ${format(pkg.to)}`, + }); + } } - private async shouldUpdateNpmSource(source: NpmSource, scope: InstalledSourceScope): Promise { + private async getGitHead(targetDir: string): Promise { + if (!existsSync(targetDir)) return undefined; + try { + const head = await this.runCommandCapture("git", ["rev-parse", "HEAD"], { + cwd: targetDir, + timeoutMs: NETWORK_TIMEOUT_MS, + }); + return head.trim() || undefined; + } catch { + return undefined; + } + } + + private async shouldUpdateNpmSource( + source: NpmSource, + scope: InstalledSourceScope, + ): Promise<{ shouldUpdate: boolean; installedVersion?: string; targetVersion?: string }> { const installedPath = this.getManagedNpmInstallPath(source, scope); const installedVersion = existsSync(installedPath) ? this.getInstalledNpmVersion(installedPath) : undefined; if (!installedVersion) { - return true; + return { shouldUpdate: true }; } try { const targetVersion = await this.getLatestNpmVersion(source.version ? source.spec : source.name, source.range); - return gt(targetVersion, installedVersion); + return { shouldUpdate: gt(targetVersion, installedVersion), installedVersion, targetVersion }; } catch { // Preserve existing update behavior when version lookup fails. - return true; + return { shouldUpdate: true, installedVersion }; } } @@ -1254,6 +1327,31 @@ export class DefaultPackageManager implements PackageManager { await this.withProgress("update", sourceLabel, message, async () => { await this.installNpmBatch(specs, scope); }); + + this.reportUnchangedNpmUpdates(sources, scope); + } + + /** + * npm can resolve stale registry metadata shortly after a publish and install the old + * version without an error. Warn when an announced update did not change the installed version. + */ + private reportUnchangedNpmUpdates(sources: NpmUpdateTarget[], scope: InstalledSourceScope): void { + for (const entry of sources) { + if (!entry.installedVersion || !entry.targetVersion) { + continue; + } + const installedPath = this.getManagedNpmInstallPath(entry.parsed, scope); + const versionAfter = this.getInstalledNpmVersion(installedPath); + if (versionAfter !== entry.installedVersion) { + continue; + } + this.emitProgress({ + type: "progress", + action: "update", + source: entry.source, + message: `${entry.source} is still ${versionAfter} (registry reports ${entry.targetVersion}); npm registry metadata may be stale, try again in a few minutes`, + }); + } } private async installNpmBatch(specs: string[], scope: InstalledSourceScope): Promise { @@ -1374,6 +1472,22 @@ export class DefaultPackageManager implements PackageManager { let installedPath = this.getNpmInstallPath(parsed, resolvedScope); const needsInstall = !existsSync(installedPath) || !(await this.installedNpmMatchesConfiguredVersion(parsed, installedPath)); + if ( + !needsInstall && + resolvedScope === "temporary" && + !parsed.pinned && + (await this.npmHasAvailableUpdate(parsed, installedPath)) + ) { + // Temporary (-e) sources have no `pi update` path, so refresh unpinned ones on load like + // temporary git sources. Like `pi update`, install `name@latest` for unversioned specs because a bare + // `npm install name` keeps the range already saved in the temporary package.json. + const spec = parsed.version ? parsed.spec : `${parsed.name}@latest`; + try { + await this.installNpm({ ...parsed, spec }, resolvedScope, true); + } catch { + // Keep the cached install if the refresh fails. + } + } if (needsInstall) { const installed = await installMissing(); if (!installed) continue; @@ -1390,8 +1504,8 @@ export class DefaultPackageManager implements PackageManager { if (!existsSync(installedPath)) { const installed = await installMissing(); if (!installed) continue; - } else if (resolvedScope === "temporary" && !parsed.pinned && !isOfflineModeEnabled()) { - await this.refreshTemporaryGitSource(parsed, resolvedSource); + } else if (resolvedScope === "temporary" && !isOfflineModeEnabled()) { + await this.refreshTemporaryGitSource(parsed, resolvedSource, installedPath); } metadata.baseDir = installedPath; metadata.packageRoot = installedPath; @@ -1471,7 +1585,7 @@ export class DefaultPackageManager implements PackageManager { if (parsed.type === "git") { return `git:${parsed.host}/${parsed.path}`; } - return `local:${this.resolvePath(parsed.path)}`; + return `local:${getPathComparisonKey(this.resolvePath(parsed.path))}`; } private getSourceMatchKeyForSettings(source: string, scope: SourceScope): string { @@ -1483,7 +1597,7 @@ export class DefaultPackageManager implements PackageManager { return `git:${parsed.host}/${parsed.path}`; } const baseDir = this.getBaseDirForScope(scope); - return `local:${this.resolvePathFromBase(parsed.path, baseDir)}`; + return `local:${getPathComparisonKey(this.resolvePathFromBase(parsed.path, baseDir))}`; } private buildNoMatchingPackageMessage(source: string, configuredPackages: PackageSource[]): string { @@ -1788,9 +1902,9 @@ export class DefaultPackageManager implements PackageManager { } if (scope) { const baseDir = this.getBaseDirForScope(scope); - return `local:${this.resolvePathFromBase(parsed.path, baseDir)}`; + return `local:${getPathComparisonKey(this.resolvePathFromBase(parsed.path, baseDir))}`; } - return `local:${this.resolvePath(parsed.path)}`; + return `local:${getPathComparisonKey(this.resolvePath(parsed.path))}`; } /** @@ -2037,20 +2151,31 @@ export class DefaultPackageManager implements PackageManager { } private async cleanAndInstallGitDependencies(targetDir: string, markerPath: string): Promise { - // Clean untracked files (extensions should be pristine). If this fails after - // deleting dependencies, repair them so the existing extension still loads. + // Clean untracked files (extensions should be pristine). On Windows, files loaded by + // another process (e.g. a native module of this extension in another pi session) cannot + // be deleted, so git clean fails. Still install dependencies for the new commit, but keep + // the marker so the next update retries the clean. + let cleaned = true; try { await this.runCommand("git", ["clean", "-fdx"], { cwd: targetDir }); } catch (error) { - await this.repairMissingGitDependencies(targetDir).catch(() => {}); - throw error; + cleaned = false; + const reason = error instanceof Error ? error.message : String(error); + this.emitProgress({ + type: "progress", + action: "update", + source: targetDir, + message: `Could not remove all untracked files in ${targetDir} (${reason}). Files may be in use by another pi session; close it and run the update again.`, + }); } const packageJsonPath = join(targetDir, "package.json"); if (existsSync(packageJsonPath)) { await this.runNpmCommand(this.getGitDependencyInstallArgs(), { cwd: targetDir }); } - rmSync(markerPath, { force: true }); + if (cleaned) { + rmSync(markerPath, { force: true }); + } } private async ensureGitRef(targetDir: string, fetchArgs: string[], ref: string): Promise { @@ -2081,11 +2206,22 @@ export class DefaultPackageManager implements PackageManager { await this.cleanAndInstallGitDependencies(targetDir, markerPath); } - private async refreshTemporaryGitSource(source: GitSource, sourceStr: string): Promise { + private async refreshTemporaryGitSource(source: GitSource, sourceStr: string, installedPath: string): Promise { if (isOfflineModeEnabled()) { return; } try { + // A ref can name a moving branch, so refresh refs too. Skip the network call only when the + // ref is a commit hash that is already checked out. + if (source.ref && /^[0-9a-f]{7,64}$/i.test(source.ref)) { + const head = await this.runCommandCapture("git", ["rev-parse", "HEAD"], { + cwd: installedPath, + timeoutMs: NETWORK_TIMEOUT_MS, + }); + if (head.trim().toLowerCase().startsWith(source.ref.toLowerCase())) { + return; + } + } await this.withProgress("pull", sourceStr, `Refreshing ${sourceStr}...`, async () => { await this.updateGit(source, "temporary"); }); @@ -2529,7 +2665,7 @@ export class DefaultPackageManager implements PackageManager { const userAgentsSkillsDir = join(getHomeDir(), ".agents", "skills"); const projectTrusted = this.settingsManager.isProjectTrusted(); const projectAgentsSkillDirs = projectTrusted - ? collectAncestorAgentsSkillDirs(this.cwd).filter((dir) => resolve(dir) !== resolve(userAgentsSkillsDir)) + ? collectAncestorAgentsSkillDirs(this.cwd).filter((dir) => !pathsEqual(dir, userAgentsSkillsDir)) : []; const addResources = ( @@ -2718,9 +2854,9 @@ export class DefaultPackageManager implements PackageManager { const seen = new Set(); return resolved.filter((entry) => { - const canonicalPath = canonicalizePath(entry.path); - if (seen.has(canonicalPath)) return false; - seen.add(canonicalPath); + const pathKey = getResourcePathKey(entry.path); + if (seen.has(pathKey)) return false; + seen.add(pathKey); return true; }); }; diff --git a/packages/coding-agent/src/core/pi-ai-schemas.ts b/packages/coding-agent/src/core/pi-ai-schemas.ts new file mode 100644 index 000000000..694aa1fe6 --- /dev/null +++ b/packages/coding-agent/src/core/pi-ai-schemas.ts @@ -0,0 +1,40 @@ +/** + * pi-ai's configuration schemas (built with pi-schema) as TypeBox schemas, for the coding agent's own TypeBox-built + * configuration schemas (models.json, settings.json): the same JSON, typed for TypeBox's `Static`. + */ + +import { + type ProviderCompat, + ProviderCompatSchema as ProviderCompatJsonSchema, +} from "@earendil-works/pi-ai/providers/compat-schema"; +import { + type ModelCost, + ModelCostSchema as ModelCostJsonSchema, + type ModelInputLimits, + ModelInputLimitsSchema as ModelInputLimitsJsonSchema, + type ModelInputModality, + ModelInputModalitySchema as ModelInputModalityJsonSchema, + type ModelPromptCache, + ModelPromptCacheSchema as ModelPromptCacheJsonSchema, + type ModelThinkingLevel, + ModelThinkingLevelSchema as ModelThinkingLevelJsonSchema, + type ThinkingLevelMap, + ThinkingLevelMapSchema as ThinkingLevelMapJsonSchema, +} from "@earendil-works/pi-ai/providers/model-schema"; +import { Type } from "typebox"; + +export const ProviderCompatSchema = Type.Unsafe(ProviderCompatJsonSchema); +export const ModelCostSchema = Type.Unsafe(ModelCostJsonSchema); +export const ModelInputLimitsSchema = Type.Unsafe(ModelInputLimitsJsonSchema); +export const ModelInputModalitySchema = Type.Unsafe(ModelInputModalityJsonSchema); +export const ModelPromptCacheSchema = Type.Unsafe(ModelPromptCacheJsonSchema); +export const ModelThinkingLevelSchema = Type.Unsafe(ModelThinkingLevelJsonSchema); +export const ThinkingLevelMapSchema = Type.Unsafe(ThinkingLevelMapJsonSchema); + +/** `ModelCostSchema` with every field optional, for model overrides that change some of the rates. */ +export const PartialModelCostSchema = Type.Unsafe>(withoutRequired(ModelCostJsonSchema)); + +function withoutRequired(schema: object): object { + const { required: _required, ...rest } = schema as { required?: unknown }; + return rest; +} diff --git a/packages/coding-agent/src/core/project-trust.ts b/packages/coding-agent/src/core/project-trust.ts index efb14be4f..3920922b8 100644 --- a/packages/coding-agent/src/core/project-trust.ts +++ b/packages/coding-agent/src/core/project-trust.ts @@ -49,7 +49,7 @@ export async function resolveProjectTrusted(options: ResolveProjectTrustedOption return options.trustOverride; } if (!hasTrustRequiringProjectResources(options.cwd)) { - return true; + return options.trustStore.get(options.cwd) ?? true; } if (options.extensionsResult) { diff --git a/packages/coding-agent/src/core/resource-loader.ts b/packages/coding-agent/src/core/resource-loader.ts index 5957abf27..bb0740e49 100644 --- a/packages/coding-agent/src/core/resource-loader.ts +++ b/packages/coding-agent/src/core/resource-loader.ts @@ -8,7 +8,7 @@ import type { ResourceDiagnostic } from "./diagnostics.ts"; export type { ResourceCollision, ResourceDiagnostic } from "./diagnostics.ts"; -import { canonicalizePath, isLocalPath, resolvePath } from "../utils/paths.ts"; +import { canonicalizePath, isLocalPath, isPathWithin, pathsEqual, resolvePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; import { createEventBus, type EventBus } from "./event-bus.ts"; import { @@ -39,6 +39,7 @@ import { loadSkills } from "./skills.ts"; import { BUILTIN_PATH_PREFIX, createSourceInfo, + getResourcePathKey, getSyntheticPathSource, isSyntheticPath, type SourceInfo, @@ -246,12 +247,12 @@ function findShadowedContextFile(cwd: string): string | undefined { const mainRepoRoot = dirname(commonGitDir); // False for an ordinary repo, where the two are the same dir, and for a sibling // worktree (`git worktree add ../feat`), whose main repo is not an ancestor. - if (!worktreeRoot.startsWith(`${mainRepoRoot}${sep}`)) return undefined; + if (!isPathWithin(mainRepoRoot, worktreeRoot) || pathsEqual(worktreeRoot, mainRepoRoot)) return undefined; // dirname of the common git dir is the main worktree root only when that dir is // itself checked out from the same repo. In a bare layout (`proj/.bare` + // `proj/main`) it is just the directory holding `.bare`, which tracks nothing; a // submodule's gitdir has no `commondir`, so it lands under `.git/modules`. - if (canonicalizePath(join(mainRepoRoot, ".git")) !== commonGitDir) return undefined; + if (!pathsEqual(canonicalizePath(join(mainRepoRoot, ".git")), commonGitDir)) return undefined; const worktreeContextFile = loadContextFileFromDir(worktreeRoot); return worktreeContextFile ? join(mainRepoRoot, basename(worktreeContextFile.path)) : undefined; } @@ -284,7 +285,7 @@ export function loadProjectContextFiles(options: { const isShadowed = shadowedContextFile !== undefined && contextFile !== null && - join(canonicalizePath(currentDir), basename(contextFile.path)) === shadowedContextFile; + pathsEqual(join(canonicalizePath(currentDir), basename(contextFile.path)), shadowedContextFile); if (contextFile && !isShadowed && !seenPaths.has(contextFile.path)) { ancestorContextFiles.unshift(contextFile); seenPaths.add(contextFile.path); @@ -750,6 +751,10 @@ export class DefaultResourceLoader implements ResourceLoader { return isSyntheticPath(path) ? path : resolvePath(path, this.cwd, { normalizeUnicodeSpaces: true }); } + private getExtensionPathKey(path: string): string { + return getResourcePathKey(this.resolveExtensionLoadPath(path)); + } + /** Load extension paths: files from disk and `builtin:` paths from the built-in extensions. */ private async loadExtensionPaths(paths: string[], runtime?: ExtensionRuntime): Promise { const isBuiltinPath = (path: string) => path.startsWith(BUILTIN_PATH_PREFIX); @@ -792,26 +797,26 @@ export class DefaultResourceLoader implements ResourceLoader { const preloadedByPath = new Map( preloaded .filter((extension) => !extension.path.startsWith(" [extension.resolvedPath, extension]), + .map((extension) => [getResourcePathKey(extension.resolvedPath), extension]), ); const failedPreloadPaths = new Set( - (preTrustExtensions?.errors ?? []).map((error) => this.resolveExtensionLoadPath(error.path)), + (preTrustExtensions?.errors ?? []).map((error) => this.getExtensionPathKey(error.path)), ); const remainingPaths = extensionPaths.filter((path) => { - const resolvedPath = this.resolveExtensionLoadPath(path); - return !preloadedByPath.has(resolvedPath) && !failedPreloadPaths.has(resolvedPath); + const pathKey = this.getExtensionPathKey(path); + return !preloadedByPath.has(pathKey) && !failedPreloadPaths.has(pathKey); }); const remainingExtensions = await this.loadExtensionPaths(remainingPaths, preTrustExtensions?.runtime); const loadedByPath = new Map(preloadedByPath); for (const extension of remainingExtensions.extensions) { - loadedByPath.set(extension.resolvedPath, extension); + loadedByPath.set(getResourcePathKey(extension.resolvedPath), extension); } const inlineExtensions = preTrustExtensions ? { extensions: preloaded.filter((extension) => extension.path.startsWith(" loadedByPath.get(this.resolveExtensionLoadPath(path))) + .map((path) => loadedByPath.get(this.getExtensionPathKey(path))) .filter((extension): extension is Extension => extension !== undefined); orderedExtensions.push(...inlineExtensions.extensions); @@ -980,11 +985,7 @@ export class DefaultResourceLoader implements ResourceLoader { const normalizedResourcePath = resolve(resourcePath); if (extraSourceInfos) { for (const [sourcePath, sourceInfo] of extraSourceInfos.entries()) { - const normalizedSourcePath = resolve(sourcePath); - if ( - normalizedResourcePath === normalizedSourcePath || - normalizedResourcePath.startsWith(`${normalizedSourcePath}${sep}`) - ) { + if (isPathWithin(sourcePath, normalizedResourcePath)) { return { ...sourceInfo, path: resourcePath }; } } @@ -997,11 +998,7 @@ export class DefaultResourceLoader implements ResourceLoader { } for (const [sourcePath, metadata] of metadataByPath.entries()) { - const normalizedSourcePath = resolve(sourcePath); - if ( - normalizedResourcePath === normalizedSourcePath || - normalizedResourcePath.startsWith(`${normalizedSourcePath}${sep}`) - ) { + if (isPathWithin(sourcePath, normalizedResourcePath)) { return createSourceInfo(resourcePath, metadata); } } @@ -1062,9 +1059,9 @@ export class DefaultResourceLoader implements ResourceLoader { for (const p of [...primary, ...additional]) { const resolved = this.resolveResourcePath(p); - const canonicalPath = canonicalizePath(resolved); - if (seen.has(canonicalPath)) continue; - seen.add(canonicalPath); + const pathKey = getResourcePathKey(resolved); + if (seen.has(pathKey)) continue; + seen.add(pathKey); merged.push(resolved); } diff --git a/packages/coding-agent/src/core/sdk.ts b/packages/coding-agent/src/core/sdk.ts index 0bf874fff..dbab1d1d9 100644 --- a/packages/coding-agent/src/core/sdk.ts +++ b/packages/coding-agent/src/core/sdk.ts @@ -4,7 +4,7 @@ import type { ModelsSimpleStreamOptions } from "@earendil-works/pi-ai"; import { clampThinkingLevel, type Message, type Model, streamSimple } from "@earendil-works/pi-ai/compat"; import { getAgentDir } from "../config.ts"; import { resolvePath } from "../utils/paths.ts"; -import { AgentSession } from "./agent-session.ts"; +import { AgentSession, markRuntimeStreamFunction } from "./agent-session.ts"; import { formatNoModelsAvailableMessage } from "./auth-guidance.ts"; import { CacheWarmer } from "./cache-warmer.ts"; import { getClientIdentity } from "./client-identity.ts"; @@ -436,6 +436,8 @@ export async function createAgentSession(options: CreateAgentSessionOptions = {} thinkingBudgets: settingsManager.getThinkingBudgets(), maxRetryDelayMs: settingsManager.getProviderRetrySettings().maxRetryDelayMs, }); + // This stream resolves auth through modelRuntime, so summaries must not pre-resolve it (#11520). + markRuntimeStreamFunction(agent.streamFunction, modelRuntime); // Restore missing settings metadata for older sessions. if (hasExistingSession) { diff --git a/packages/coding-agent/src/core/session-manager.ts b/packages/coding-agent/src/core/session-manager.ts index 89431f2a3..20b6edaa9 100644 --- a/packages/coding-agent/src/core/session-manager.ts +++ b/packages/coding-agent/src/core/session-manager.ts @@ -30,7 +30,7 @@ import { basename, join, resolve } from "path"; import { createInterface } from "readline"; import { StringDecoder } from "string_decoder"; import { APP_NAME, getAgentDir as getDefaultAgentDir, getSessionsDir } from "../config.ts"; -import { normalizePath, resolvePath } from "../utils/paths.ts"; +import { normalizePath, pathsEqual, resolvePath } from "../utils/paths.ts"; import { type BashExecutionMessage, type CustomMessage, @@ -516,7 +516,8 @@ export function buildContextEntries( * If leafId is provided, walks from that entry to root. * Handles compaction and branch summaries along the path. */ -function projectContextEntry(entry: SessionEntry, edit: ContextEditEntry | undefined): AgentMessage[] { +/** Project one entry into model-visible messages, applying its latest context edit if any. */ +export function projectContextEntry(entry: SessionEntry, edit: ContextEditEntry | undefined): AgentMessage[] { const messages = sessionEntryToContextMessages(entry); if (!edit) return messages; const replacement = edit.replacement; @@ -742,7 +743,7 @@ function getSessionHeaderCwd(header: SessionHeader): string | undefined { } function sessionCwdMatches(cwd: string | undefined, resolvedCwd: string): boolean { - return cwd !== undefined && cwd !== "" && resolvePath(cwd) === resolvedCwd; + return cwd !== undefined && cwd !== "" && pathsEqual(resolvePath(cwd), resolvedCwd); } /** Exported for testing */ @@ -1146,7 +1147,7 @@ export class SessionManager { } usesDefaultSessionDir(): boolean { - return this.sessionDir === getDefaultSessionDirPath(this.cwd); + return pathsEqual(this.sessionDir, getDefaultSessionDirPath(this.cwd)); } getSessionId(): string { @@ -1802,7 +1803,7 @@ export class SessionManager { */ static continueRecent(cwd: string, sessionDir?: string): SessionManager { const dir = sessionDir ? normalizePath(sessionDir) : getDefaultSessionDir(cwd); - const filterCwd = sessionDir !== undefined && dir !== getDefaultSessionDirPath(cwd); + const filterCwd = sessionDir !== undefined && !pathsEqual(dir, getDefaultSessionDirPath(cwd)); const mostRecent = findMostRecentSession(dir, filterCwd ? cwd : undefined); if (mostRecent) { return new SessionManager(cwd, dir, mostRecent, true); @@ -1888,7 +1889,7 @@ export class SessionManager { */ static findById(cwd: string, id: string, sessionDir?: string): string | undefined { const dir = sessionDir ? normalizePath(sessionDir) : getDefaultSessionDir(cwd); - const filterCwd = sessionDir !== undefined && dir !== getDefaultSessionDirPath(cwd); + const filterCwd = sessionDir !== undefined && !pathsEqual(dir, getDefaultSessionDirPath(cwd)); const resolvedCwd = resolvePath(cwd); try { @@ -1919,7 +1920,7 @@ export class SessionManager { signal?: AbortSignal, ): Promise { const dir = sessionDir ? normalizePath(sessionDir) : getDefaultSessionDir(cwd); - const filterCwd = sessionDir !== undefined && dir !== getDefaultSessionDirPath(cwd); + const filterCwd = sessionDir !== undefined && !pathsEqual(dir, getDefaultSessionDirPath(cwd)); const resolvedCwd = resolvePath(cwd); const includeSession = (session: SessionInfo) => !filterCwd || sessionCwdMatches(session.cwd, resolvedCwd); const progress: SessionListProgress | undefined = onProgress diff --git a/packages/coding-agent/src/core/settings-schema.ts b/packages/coding-agent/src/core/settings-schema.ts index 87a3b71a5..1ef797338 100644 --- a/packages/coding-agent/src/core/settings-schema.ts +++ b/packages/coding-agent/src/core/settings-schema.ts @@ -1,5 +1,5 @@ -import { ModelThinkingLevelSchema } from "@earendil-works/pi-ai/providers/model-schema"; import { type Static, type TSchemaOptions, Type } from "typebox"; +import { ModelThinkingLevelSchema } from "./pi-ai-schemas.ts"; import { SETTINGS_DEFAULTS } from "./settings-defaults.ts"; function nonNegativeSafeInteger(options: { default?: number } = {}) { @@ -283,6 +283,13 @@ export const SettingsSchema = Type.Object( 'Command used for npm package lookup and installation, in argv form such as ["mise", "exec", "node@20", "--", "npm"].', }), ), + selfUpdateCommand: Type.Optional( + Type.Array(Type.String(), { + minItems: 1, + description: + 'Command `pi update` runs to update Pi itself, in argv form such as ["mise", "use", "-g", "npm:{packageName}@{version}"]. Placeholders: {packageName}, {version}, {installSpec}. PI_SELF_UPDATE_COMMAND takes precedence. Global setting only.', + }), + ), collapseChangelog: Type.Optional( Type.Boolean({ description: "Show the condensed changelog after update; use /changelog for the full changelog.", diff --git a/packages/coding-agent/src/core/skills.ts b/packages/coding-agent/src/core/skills.ts index 3fbf2b926..48df4536b 100644 --- a/packages/coding-agent/src/core/skills.ts +++ b/packages/coding-agent/src/core/skills.ts @@ -3,9 +3,9 @@ import ignore from "ignore"; import { basename, dirname, join, relative, resolve, sep } from "path"; import { CONFIG_DIR_NAME, getAgentDir } from "../config.ts"; import { parseFrontmatter } from "../utils/frontmatter.ts"; -import { canonicalizePath, resolvePath } from "../utils/paths.ts"; +import { isPathWithin, resolvePath } from "../utils/paths.ts"; import type { ResourceDiagnostic } from "./diagnostics.ts"; -import { createSyntheticSourceInfo, type SourceInfo } from "./source-info.ts"; +import { createSyntheticSourceInfo, getResourcePathKey, type SourceInfo } from "./source-info.ts"; /** Max name length per spec */ const MAX_NAME_LENGTH = 64; @@ -369,7 +369,7 @@ export function formatSkillsForPrompt(skills: Skill[], fileReadTool: "read" | "b : fileReadTool === "bash" ? "Use bash to load a skill's file when the task matches its description." : "Load a skill's file when the task matches its description.", - "When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.", + "Resolve relative paths against the skill's directory (parent of SKILL.md). Use absolute paths in tool calls.", "", "", ]; @@ -427,7 +427,7 @@ export function loadSkills(options: LoadSkillsOptions): LoadSkillsResult { allDiagnostics.push(...result.diagnostics); for (const skill of result.skills) { // Resolve symlinks to detect duplicate files - const realPath = canonicalizePath(skill.filePath); + const realPath = getResourcePathKey(skill.filePath); // Skip silently if we've already loaded this exact file (via symlink) if (realPathSet.has(realPath)) { @@ -462,19 +462,10 @@ export function loadSkills(options: LoadSkillsOptions): LoadSkillsResult { const userSkillsDir = join(resolvedAgentDir, "skills"); const projectSkillsDir = resolve(resolvedCwd, CONFIG_DIR_NAME, "skills"); - const isUnderPath = (target: string, root: string): boolean => { - const normalizedRoot = resolve(root); - if (target === normalizedRoot) { - return true; - } - const prefix = normalizedRoot.endsWith(sep) ? normalizedRoot : `${normalizedRoot}${sep}`; - return target.startsWith(prefix); - }; - const getSource = (resolvedPath: string): "user" | "project" | "path" => { if (!includeDefaults) { - if (isUnderPath(resolvedPath, userSkillsDir)) return "user"; - if (isUnderPath(resolvedPath, projectSkillsDir)) return "project"; + if (isPathWithin(userSkillsDir, resolvedPath)) return "user"; + if (isPathWithin(projectSkillsDir, resolvedPath)) return "project"; } return "path"; }; diff --git a/packages/coding-agent/src/core/slash-commands.ts b/packages/coding-agent/src/core/slash-commands.ts index dc98f1d43..97b2caa6f 100644 --- a/packages/coding-agent/src/core/slash-commands.ts +++ b/packages/coding-agent/src/core/slash-commands.ts @@ -23,7 +23,7 @@ export const BUILTIN_SLASH_COMMANDS: ReadonlyArray = [ { name: "thinking", description: "Set thinking level", argumentHint: "" }, { name: "scoped-models", description: "Enable/disable models for Ctrl+P cycling" }, { name: "export", description: "Export session (HTML default, or specify path: .html/.jsonl)" }, - { name: "import", description: "Import and resume a session from a JSONL file" }, + { name: "import", description: "Import a JSONL session, preserving its ID and working directory" }, { name: "share", description: "Share session via a Radius artifact or a GitHub gist" }, { name: "bug", description: "Report a bug to the Pi developers", argumentHint: "" }, { name: "copy", description: "Copy last agent message to clipboard" }, @@ -38,7 +38,7 @@ export const BUILTIN_SLASH_COMMANDS: ReadonlyArray = [ { name: "logout", description: "Remove provider authentication" }, { name: "new", description: "Start a new session" }, { name: "compact", description: "Manually compact the session context" }, - { name: "resume", description: "Resume a different session" }, + { name: "resume", description: "Resume another session" }, { name: "reload", description: "Reload keybindings, extensions, skills, prompts, themes, and context files" }, { name: "quit", description: `Quit ${APP_NAME}` }, ]; diff --git a/packages/coding-agent/src/core/source-info.ts b/packages/coding-agent/src/core/source-info.ts index e73a0471d..716e8c069 100644 --- a/packages/coding-agent/src/core/source-info.ts +++ b/packages/coding-agent/src/core/source-info.ts @@ -1,3 +1,4 @@ +import { canonicalizePath, getPathComparisonKey } from "../utils/paths.ts"; import type { PathMetadata } from "./package-manager.ts"; export type SourceScope = "user" | "project" | "temporary"; @@ -41,6 +42,11 @@ export function isSyntheticPath(path: string): boolean { return path.startsWith(BUILTIN_PATH_PREFIX) || path.startsWith("<"); } +/** Key for deduplicating resource paths: synthetic paths as-is, file paths via realpath and platform casing. */ +export function getResourcePathKey(path: string): string { + return isSyntheticPath(path) ? path : getPathComparisonKey(canonicalizePath(path)); +} + export function createSourceInfo(path: string, metadata: PathMetadata): SourceInfo { return { path, diff --git a/packages/coding-agent/src/core/system-prompt.ts b/packages/coding-agent/src/core/system-prompt.ts index 49bcb58ee..e51e14fe4 100644 --- a/packages/coding-agent/src/core/system-prompt.ts +++ b/packages/coding-agent/src/core/system-prompt.ts @@ -153,20 +153,19 @@ export function buildSystemPromptSections(input: BuildSystemPromptOptions): Syst promptSections.preamble = customPrompt; } else { promptSections.preamble = - "You are an expert coding assistant operating inside pi, a coding agent harness. You help users by reading files, executing commands, editing code, and writing new files."; + "You are a coding assistant running inside pi, a coding agent harness. Help users inspect files, run commands, and modify code."; const visibleTools = declaredTools.filter((name) => !!toolSnippets[name]); const tools = visibleTools.length > 0 ? visibleTools.map((name) => `- ${name}: ${toolSnippets[name]}`).join("\n") : "(none)"; - promptSections.tools = `${tools}\n\nIn addition to the tools above, you may have access to other custom tools depending on the project.`; + promptSections.tools = `${tools}\n\nAdditional tools may be available.`; promptSections.rules = buildRules(declaredTools, toolGuidelines, promptGuidelines); - promptSections.docs = `Pi documentation (read only when the user asks about pi itself, its SDK, extensions, themes, skills, or TUI): + promptSections.docs = `Pi documentation (read only when the user asks about the harness itself, its SDK, extensions, themes, skills, or TUI): - Main documentation: ${getReadmePath()} - Additional docs: ${getDocsPath()} - Examples: ${getExamplesPath()} (extensions, custom tools, SDK) - When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory -- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), pi packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md) -- When working on pi topics, read the docs and examples, and follow .md cross-references before implementing -- Always read pi .md files completely and follow links to related docs (e.g., tui.md for TUI API details)`; +- When asked about: extensions (docs/extensions.md, examples/extensions/), themes (docs/themes.md), skills (docs/skills.md), prompt templates (docs/prompt-templates.md), TUI components (docs/tui.md), keybindings (docs/keybindings.md), SDK integrations (docs/sdk.md), custom providers (docs/custom-provider.md), adding models (docs/models.md), packages (docs/packages.md), environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md), codemode scripts and non-LLM models such as classifiers and image models (docs/codemode.md) +- Before implementing harness changes, read the relevant docs and examples in full, following related cross-references.`; } if (appendSystemPrompt) promptSections.addendum = appendSystemPrompt; diff --git a/packages/coding-agent/src/core/tools/bash.ts b/packages/coding-agent/src/core/tools/bash.ts index 4620e2464..2084d6f7c 100644 --- a/packages/coding-agent/src/core/tools/bash.ts +++ b/packages/coding-agent/src/core/tools/bash.ts @@ -1,7 +1,6 @@ import { constants } from "node:fs"; import { access as fsAccess } from "node:fs/promises"; import { constants as osConstants } from "node:os"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import { type Static, Type } from "typebox"; import { waitForChildProcess } from "../../utils/child-process.ts"; @@ -16,7 +15,7 @@ import { import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { OutputAccumulator } from "./output-accumulator.ts"; import { BASH_UPDATE_THROTTLE_MS, createShellRenderers } from "./renderers/bash.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult } from "./truncate.ts"; const MAX_TIMEOUT_MS = 2_147_483_647; @@ -431,7 +430,7 @@ export function createBashToolDefinition( return createShellToolDefinition(cwd, bashToolConfig, options); } -export function createBashTool(cwd: string, options?: BashToolOptions): AgentTool { +export function createBashTool(cwd: string, options?: BashToolOptions): TypeBoxAgentTool { const definition = createBashToolDefinition(cwd, options); const tool = wrapToolDefinition(definition); Object.assign(tool, { diff --git a/packages/coding-agent/src/core/tools/edit.ts b/packages/coding-agent/src/core/tools/edit.ts index 62e6fdee0..bb6aeff59 100644 --- a/packages/coding-agent/src/core/tools/edit.ts +++ b/packages/coding-agent/src/core/tools/edit.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile, writeFile as fsWriteFile } from "fs/promises"; import { type Static, Type } from "typebox"; @@ -16,7 +15,7 @@ import { import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToCwd } from "./path-utils.ts"; import { type EditRenderState, editRenderers } from "./renderers/edit.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; const replaceEditSchema = Type.Object( { @@ -43,10 +42,8 @@ const editSchema = Type.Object( export const editToolSystemPromptContribution = { snippet: "Make precise file edits with exact text replacement, including multiple disjoint edits in one call", guidelines: [ - "Use edit for precise changes (edits[].oldText must match exactly)", - "When changing multiple separate locations in one file, use one edit call with multiple entries in edits[] instead of multiple edit calls", - "Each edits[].oldText is matched against the original file, not after earlier edits are applied. Do not emit overlapping or nested edits. Merge nearby changes into one edit.", - "Keep edits[].oldText as small as possible while still being unique in the file. Do not pad with large unchanged regions.", + "Use edit for targeted changes. Batch changes to one file in a single call.", + "Each edits[].oldText must exactly and uniquely match the original file. Keep matches short and non-overlapping; merge nearby changes without including large unchanged regions.", ], } as const; @@ -215,6 +212,6 @@ export function createEditToolDefinition( }; } -export function createEditTool(cwd: string, options?: EditToolOptions): AgentTool { +export function createEditTool(cwd: string, options?: EditToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createEditToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/find.ts b/packages/coding-agent/src/core/tools/find.ts index 8d44c491e..77ba1b356 100644 --- a/packages/coding-agent/src/core/tools/find.ts +++ b/packages/coding-agent/src/core/tools/find.ts @@ -1,5 +1,4 @@ import { createInterface } from "node:readline"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import path from "path"; import { type Static, Type } from "typebox"; @@ -7,7 +6,7 @@ import { ensureTool } from "../../utils/tools-manager.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { pathExists, resolveToCwd } from "./path-utils.ts"; import { findRenderers } from "./renderers/find.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; /** Relativize a find result against the search root and normalize it to posix separators. */ @@ -313,6 +312,6 @@ export function createFindToolDefinition( }; } -export function createFindTool(cwd: string, options?: FindToolOptions): AgentTool { +export function createFindTool(cwd: string, options?: FindToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createFindToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/grep.ts b/packages/coding-agent/src/core/tools/grep.ts index 77d830cd1..e449e0898 100644 --- a/packages/coding-agent/src/core/tools/grep.ts +++ b/packages/coding-agent/src/core/tools/grep.ts @@ -1,6 +1,5 @@ import { readFile as fsReadFile, stat as fsStat } from "node:fs/promises"; import { createInterface } from "node:readline"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { spawn } from "child_process"; import path from "path"; import { type Static, Type } from "typebox"; @@ -8,7 +7,7 @@ import { ensureTool } from "../../utils/tools-manager.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { resolveToCwd } from "./path-utils.ts"; import { grepRenderers } from "./renderers/grep.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, @@ -320,6 +319,6 @@ export function createGrepToolDefinition( }; } -export function createGrepTool(cwd: string, options?: GrepToolOptions): AgentTool { +export function createGrepTool(cwd: string, options?: GrepToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createGrepToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/ls.ts b/packages/coding-agent/src/core/tools/ls.ts index 558ff9f9f..4f5efc8a7 100644 --- a/packages/coding-agent/src/core/tools/ls.ts +++ b/packages/coding-agent/src/core/tools/ls.ts @@ -1,11 +1,10 @@ import { readdir as fsReaddir, stat as fsStat } from "node:fs/promises"; -import type { AgentTool } from "@earendil-works/pi-agent-core"; import nodePath from "path"; import { type Static, Type } from "typebox"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { pathExists, resolveToCwd } from "./path-utils.ts"; import { lsRenderers } from "./renderers/ls.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; const lsSchema = Type.Object({ @@ -170,6 +169,6 @@ export function createLsToolDefinition( }; } -export function createLsTool(cwd: string, options?: LsToolOptions): AgentTool { +export function createLsTool(cwd: string, options?: LsToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createLsToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/read.ts b/packages/coding-agent/src/core/tools/read.ts index ebe2c2517..8fb93a679 100644 --- a/packages/coding-agent/src/core/tools/read.ts +++ b/packages/coding-agent/src/core/tools/read.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import type { Api, ImageContent, Model, ModelImageResizeOptions, TextContent } from "@earendil-works/pi-ai"; import { constants } from "fs"; import { access as fsAccess, readFile as fsReadFile } from "fs/promises"; @@ -8,7 +7,7 @@ import { detectSupportedImageMimeTypeFromFile } from "../../utils/mime.ts"; import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { resolveReadPathAsync } from "./path-utils.ts"; import { readRenderers } from "./renderers/read.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult, truncateHead } from "./truncate.ts"; const readSchema = Type.Object({ @@ -219,6 +218,6 @@ export function createReadToolDefinition( }; } -export function createReadTool(cwd: string, options?: ReadToolOptions): AgentTool { +export function createReadTool(cwd: string, options?: ReadToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createReadToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/tools/render-utils.ts b/packages/coding-agent/src/core/tools/render-utils.ts index c0fca375d..1bde70bc3 100644 --- a/packages/coding-agent/src/core/tools/render-utils.ts +++ b/packages/coding-agent/src/core/tools/render-utils.ts @@ -1,20 +1,15 @@ -import * as os from "node:os"; import { pathToFileURL } from "node:url"; import type { ImageContent, TextContent } from "@earendil-works/pi-ai"; import { getCapabilities, getImageDimensions, hyperlink, imageFallback } from "@earendil-works/pi-tui"; import type { Theme } from "../../modes/interactive/theme/theme.ts"; import { stripAnsi } from "../../utils/ansi.ts"; -import { resolvePath } from "../../utils/paths.ts"; +import { resolvePath, shortenHomePath } from "../../utils/paths.ts"; import { sanitizeBinaryOutput } from "../../utils/shell.ts"; import type { ToolRenderContext } from "../extensions/types.ts"; export function shortenPath(path: unknown): string { if (typeof path !== "string") return ""; - const home = os.homedir(); - if (path.startsWith(home)) { - return `~${path.slice(home.length)}`; - } - return path; + return shortenHomePath(path); } export function linkPath(styledText: string, rawPath: string, cwd: string): string { diff --git a/packages/coding-agent/src/core/tools/renderers/write.ts b/packages/coding-agent/src/core/tools/renderers/write.ts index 7c8f32636..8e2338a7d 100644 --- a/packages/coding-agent/src/core/tools/renderers/write.ts +++ b/packages/coding-agent/src/core/tools/renderers/write.ts @@ -8,7 +8,12 @@ import { Container, Text } from "@earendil-works/pi-tui"; import { keyHint } from "../../../modes/interactive/components/keybinding-hints.ts"; -import { getLanguageFromPath, highlightCode, type Theme } from "../../../modes/interactive/theme/theme.ts"; +import { + CodeHighlighter, + getLanguageFromPath, + highlightCode, + type Theme, +} from "../../../modes/interactive/theme/theme.ts"; import type { ToolDefinition, ToolRenderResultOptions } from "../../extensions/types.ts"; import { normalizeDisplayText, renderToolPath, replaceTabs, str } from "../render-utils.ts"; @@ -20,7 +25,12 @@ type WriteHighlightCache = { highlightedLines: string[]; }; class WriteCallRenderComponent extends Text { + // Line by line highlighting of a call whose content is still streaming. cache?: WriteHighlightCache; + // A streamed call highlights its first lines on every delta, which stop changing once the file is longer, and a + // complete call highlights the whole file on every update and expand toggle. Either way the row reuses the last + // highlighting; it also holds the only copy of a complete call's displayed source. + readonly highlighter = new CodeHighlighter(); constructor() { super("", 0, 0); @@ -31,11 +41,11 @@ function highlightSingleLine(line: string, lang: string): string { const highlighted = highlightCode(line, lang); return highlighted[0] ?? ""; } -function refreshWriteHighlightPrefix(cache: WriteHighlightCache): void { +function refreshWriteHighlightPrefix(cache: WriteHighlightCache, highlighter: CodeHighlighter): void { const prefixCount = Math.min(WRITE_PARTIAL_FULL_HIGHLIGHT_LINES, cache.normalizedLines.length); if (prefixCount === 0) return; const prefixSource = cache.normalizedLines.slice(0, prefixCount).join("\n"); - const prefixHighlighted = highlightCode(prefixSource, cache.lang); + const prefixHighlighted = highlighter.highlight(prefixSource, cache.lang); for (let i = 0; i < prefixCount; i++) { cache.highlightedLines[i] = prefixHighlighted[i] ?? highlightSingleLine(cache.normalizedLines[i] ?? "", cache.lang); @@ -58,12 +68,14 @@ function updateWriteHighlightCacheIncremental( cache: WriteHighlightCache | undefined, rawPath: string | null, fileContent: string, + highlighter: CodeHighlighter, ): WriteHighlightCache | undefined { const lang = rawPath ? getLanguageFromPath(rawPath) : undefined; if (!lang) return undefined; if (!cache) return rebuildWriteHighlightCacheFull(rawPath, fileContent); - if (cache.lang !== lang || cache.rawPath !== rawPath) return rebuildWriteHighlightCacheFull(rawPath, fileContent); - if (!fileContent.startsWith(cache.rawContent)) return rebuildWriteHighlightCacheFull(rawPath, fileContent); + if (cache.lang !== lang || cache.rawPath !== rawPath || !fileContent.startsWith(cache.rawContent)) { + return rebuildWriteHighlightCacheFull(rawPath, fileContent); + } if (fileContent.length === cache.rawContent.length) return cache; const deltaRaw = fileContent.slice(cache.rawContent.length); @@ -83,10 +95,10 @@ function updateWriteHighlightCacheIncremental( cache.normalizedLines.push(segments[i]); cache.highlightedLines.push(highlightSingleLine(segments[i], cache.lang)); } - refreshWriteHighlightPrefix(cache); + refreshWriteHighlightPrefix(cache, highlighter); return cache; } -function trimTrailingEmptyLines(lines: string[]): string[] { +function trimTrailingEmptyLines(lines: readonly string[]): string[] { let end = lines.length; while (end > 0 && lines[end - 1] === "") { end--; @@ -98,6 +110,7 @@ function formatWriteCall( options: ToolRenderResultOptions, theme: Theme, cache: WriteHighlightCache | undefined, + highlighter: CodeHighlighter, cwd: string, ): string { const rawPath = str(args?.file_path ?? args?.path); @@ -110,7 +123,7 @@ function formatWriteCall( } else if (fileContent) { const lang = rawPath ? getLanguageFromPath(rawPath) : undefined; const renderedLines = lang - ? (cache?.highlightedLines ?? highlightCode(replaceTabs(normalizeDisplayText(fileContent)), lang)) + ? (cache?.highlightedLines ?? highlighter.highlight(replaceTabs(normalizeDisplayText(fileContent)), lang)) : normalizeDisplayText(fileContent).split("\n"); const lines = trimTrailingEmptyLines(renderedLines); const totalLines = lines.length; @@ -149,19 +162,17 @@ export const writeRenderers: Pick, "renderCall" | "rend const fileContent = str(renderArgs?.content); const component = (context.lastComponent as WriteCallRenderComponent | undefined) ?? new WriteCallRenderComponent(); - if (fileContent !== null) { - component.cache = context.argsComplete - ? rebuildWriteHighlightCacheFull(rawPath, fileContent) - : updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent); - } else { - component.cache = undefined; - } + component.cache = + fileContent !== null && !context.argsComplete + ? updateWriteHighlightCacheIncremental(component.cache, rawPath, fileContent, component.highlighter) + : undefined; component.setText( formatWriteCall( renderArgs, { expanded: context.expanded, isPartial: context.isPartial }, theme, component.cache, + component.highlighter, context.cwd, ), ); diff --git a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts index 9a9465dec..700dfd35c 100644 --- a/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts +++ b/packages/coding-agent/src/core/tools/tool-definition-wrapper.ts @@ -1,6 +1,25 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; +import type { AgentTool, AgentToolResult, AgentToolUpdateCallback } from "@earendil-works/pi-agent-core"; +import type { Static, TSchema } from "typebox"; import type { ExtensionToolContext, ToolDefinition } from "../extensions/types.ts"; +/** + * An `AgentTool` with a TypeBox parameter schema whose `execute` and `prepareArguments` are typed by TypeBox's + * `Static`, as the built-in tools are (pi-agent-core types `params` through pi-schema, which maps only the common + * TypeBox kinds). Spreads into a `ToolDefinition` of the same schema. + */ +export type TypeBoxAgentTool = Omit< + AgentTool, + "prepareArguments" | "execute" +> & { + prepareArguments?: (args: unknown) => Static; + execute: ( + toolCallId: string, + params: Static, + signal?: AbortSignal, + onUpdate?: AgentToolUpdateCallback, + ) => Promise>; +}; + /** Creates the context for one tool call. */ export type ToolContextFactory = (toolCallId: string, signal: AbortSignal | undefined) => ExtensionToolContext; diff --git a/packages/coding-agent/src/core/tools/write.ts b/packages/coding-agent/src/core/tools/write.ts index 374089017..9e6b296dc 100644 --- a/packages/coding-agent/src/core/tools/write.ts +++ b/packages/coding-agent/src/core/tools/write.ts @@ -1,4 +1,3 @@ -import type { AgentTool } from "@earendil-works/pi-agent-core"; import { mkdir as fsMkdir, writeFile as fsWriteFile } from "fs/promises"; import { dirname } from "path"; import { type Static, Type } from "typebox"; @@ -6,7 +5,7 @@ import type { ExtensionContext, ToolDefinition } from "../extensions/types.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToCwd } from "./path-utils.ts"; import { writeRenderers } from "./renderers/write.ts"; -import { wrapToolDefinition } from "./tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "./tool-definition-wrapper.ts"; const writeSchema = Type.Object({ path: Type.String({ description: "Path to the file to write (relative or absolute)" }), @@ -92,6 +91,6 @@ export function createWriteToolDefinition( }; } -export function createWriteTool(cwd: string, options?: WriteToolOptions): AgentTool { +export function createWriteTool(cwd: string, options?: WriteToolOptions): TypeBoxAgentTool { return wrapToolDefinition(createWriteToolDefinition(cwd, options)); } diff --git a/packages/coding-agent/src/core/trust-manager.ts b/packages/coding-agent/src/core/trust-manager.ts index 3396c9a39..f0721efef 100644 --- a/packages/coding-agent/src/core/trust-manager.ts +++ b/packages/coding-agent/src/core/trust-manager.ts @@ -3,7 +3,7 @@ import { homedir } from "node:os"; import { dirname, join } from "node:path"; import lockfile from "proper-lockfile"; import { CONFIG_DIR_NAME } from "../config.ts"; -import { canonicalizePath, resolvePath } from "../utils/paths.ts"; +import { canonicalizePath, pathsEqual, resolvePath } from "../utils/paths.ts"; import { stripBom } from "../utils/text.ts"; export type ProjectTrustDecision = boolean | null; @@ -42,12 +42,19 @@ function normalizeCwd(cwd: string): string { return canonicalizePath(resolvePath(cwd)); } +/** Trust file keys naming `path`. Usually one; several on Windows when saved with different casing. */ +function findTrustFileKeys(data: TrustFile, path: string): string[] { + return Object.keys(data).filter((key) => pathsEqual(key, path)); +} + function findNearestTrustEntry(data: TrustFile, cwd: string): ProjectTrustStoreEntry | null { let currentDir = normalizeCwd(cwd); while (true) { - const value = data[currentDir]; - if (value === true || value === false) { - return { path: currentDir, decision: value }; + for (const key of findTrustFileKeys(data, currentDir)) { + const value = data[key]; + if (value === true || value === false) { + return { path: key, decision: value }; + } } const parentDir = dirname(currentDir); @@ -195,7 +202,7 @@ export function hasTrustRequiringProjectResources(cwd: string): boolean { while (true) { const agentsSkillsDir = join(currentDir, ".agents", "skills"); - if (agentsSkillsDir !== userAgentsSkillsDir && existsSync(agentsSkillsDir)) { + if (!pathsEqual(agentsSkillsDir, userAgentsSkillsDir) && existsSync(agentsSkillsDir)) { return true; } @@ -234,9 +241,10 @@ export class ProjectTrustStore { const data = readTrustFile(this.trustPath); for (const { path, decision } of decisions) { const key = normalizeCwd(path); - if (decision === null) { - delete data[key]; - } else { + for (const existingKey of findTrustFileKeys(data, key)) { + delete data[existingKey]; + } + if (decision !== null) { data[key] = decision; } } diff --git a/packages/coding-agent/src/extensions/codemode/renderer.ts b/packages/coding-agent/src/extensions/codemode/renderer.ts index 8d2937351..660803e68 100644 --- a/packages/coding-agent/src/extensions/codemode/renderer.ts +++ b/packages/coding-agent/src/extensions/codemode/renderer.ts @@ -12,7 +12,7 @@ import type { ToolDefinition } from "../../core/extensions/types.ts"; import { getTextOutput, replaceTabs, startToolTimer, str, toolTimerText } from "../../core/tools/render-utils.ts"; import { keyHint } from "../../modes/interactive/components/keybinding-hints.ts"; import { VisualLinePreview } from "../../modes/interactive/components/visual-truncate.ts"; -import { highlightCode, type Theme } from "../../modes/interactive/theme/theme.ts"; +import { CodeHighlighter, type Theme } from "../../modes/interactive/theme/theme.ts"; import type { CodemodeNestedCall, CodemodeToolDetails } from "./tool.ts"; const CODE_PREVIEW_LINES = 10; @@ -21,6 +21,12 @@ const OUTPUT_PREVIEW_LINES = 5; const COLLAPSED_ARGS_CHARS = 80; const SCRIPT_HEADER = /^Script (completed|failed)\nWall time [\d.]+ seconds\nOutput:\n$/; +/** Renderer state of a codemode row. */ +interface CodemodeRenderState { + // Progress, timer ticks and expanding redraw the call, so the row keeps the script's highlighting. + scriptHighlighter?: CodeHighlighter; +} + function expandHint(theme: Theme, hidden: number, noun: string): string { return `${theme.fg("muted", `... (${hidden} more ${noun},`)} ${keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`; } @@ -79,7 +85,11 @@ export const codemodeRenderers: Pick< } component.addChild(new Text(title, 0, 0)); if (code) { - const highlighted = highlightCode(replaceTabs(code.replace(/\r/g, "").trimEnd()), "javascript").join("\n"); + const state = context.state as CodemodeRenderState; + state.scriptHighlighter ??= new CodeHighlighter(); + const highlighted = state.scriptHighlighter + .highlight(replaceTabs(code.replace(/\r/g, "").trimEnd()), "javascript") + .join("\n"); component.addChild( context.expanded ? new Text(highlighted, 0, 0) diff --git a/packages/coding-agent/src/extensions/codemode/tool.ts b/packages/coding-agent/src/extensions/codemode/tool.ts index e28ea999f..648ba6109 100644 --- a/packages/coding-agent/src/extensions/codemode/tool.ts +++ b/packages/coding-agent/src/extensions/codemode/tool.ts @@ -32,7 +32,7 @@ import type { ToolDefinition, ToolLoadout, ToolLoadoutChanges, ToolNamespace } f import type { ModelRegistry } from "../../core/model-registry.ts"; import { SETTINGS_DEFAULTS } from "../../core/settings-defaults.ts"; import type { CodemodeMode } from "../../core/settings-manager.ts"; -import { wrapToolDefinition } from "../../core/tools/tool-definition-wrapper.ts"; +import { type TypeBoxAgentTool, wrapToolDefinition } from "../../core/tools/tool-definition-wrapper.ts"; import { loadCodemodeExecutor } from "./execute.lazy.ts"; import { codemodeRenderers } from "./renderer.ts"; @@ -422,7 +422,7 @@ export function createCodemodeToolDefinition( export function createCodemodeTool( tools: readonly AgentTool[] = [], options: CodemodeToolOptions = {}, -): AgentTool { +): TypeBoxAgentTool { const definition = createCodemodeToolDefinition(options); const tool = wrapToolDefinition(definition); Object.assign(tool, { diff --git a/packages/coding-agent/src/extensions/mcp/cli.ts b/packages/coding-agent/src/extensions/mcp/cli.ts index 75cdf1f08..81d28ad66 100644 --- a/packages/coding-agent/src/extensions/mcp/cli.ts +++ b/packages/coding-agent/src/extensions/mcp/cli.ts @@ -35,8 +35,8 @@ const HELP = `${chalk.bold("Usage:")} ${APP_NAME} mcp add [options] --url ${APP_NAME} mcp remove [-l] ${APP_NAME} mcp list [--json] - ${APP_NAME} mcp login [--timeout ] - ${APP_NAME} mcp logout + ${APP_NAME} mcp login [-l] [--timeout ] + ${APP_NAME} mcp logout [-l] Configure and check MCP servers and sign in to OAuth servers without starting a session. Reads ~/${CONFIG_DIR_NAME}/agent/mcp.json and, in trusted projects, ${CONFIG_DIR_NAME}/mcp.json. @@ -48,7 +48,7 @@ Commands: login Sign in through the browser logout Delete the stored OAuth credentials -Options for add and remove: +Options for add, remove, login, and logout: -l, --local Use ${CONFIG_DIR_NAME}/mcp.json in the current project instead of the global file Options for add: @@ -65,6 +65,8 @@ Options for add: Fixed OAuth callback port --oauth-client-name Client name sent when registering with the OAuth server + --oauth-client-registration + auto (default), cimd, or dcr --exposure codemode (default), deferred, direct, or hidden --description What the server offers, shown in the system prompt @@ -151,7 +153,8 @@ function parseOptions( const values = new Map(); const lists = new Map(); for (let index = 0; index < args.length; index++) { - const arg = OPTION_ALIASES.get(args[index]) ?? args[index]; + const typed = args[index]; + const arg = OPTION_ALIASES.get(typed) ?? typed; if (arg === "--" || positional.length >= maxPositionals) { positional.push(...args.slice(arg === "--" ? index + 1 : index)); break; @@ -163,7 +166,7 @@ function parseOptions( const name = arg.slice(2); const kind = known[name]; if (!kind) { - error(`Unknown option ${arg}.\n${HELP_HINT}`); + error(`Unknown option ${typed}.\n${HELP_HINT}`); return undefined; } if (kind === "flag") { @@ -217,20 +220,30 @@ export async function runMcpCommand(args: string[], options: McpCommandOptions): } case "login": case "logout": { - const parsed = parseOptions(rest, command === "login" ? { timeout: "value" } : {}, error); + const parsed = parseOptions( + rest, + command === "login" ? { local: "flag", timeout: "value" } : { local: "flag" }, + error, + ); if (!parsed) return 1; const [name, ...extra] = parsed.positional; if (!name || extra.length > 0) { - error(`Usage: ${APP_NAME} mcp ${command} \n${HELP_HINT}`); + error(`Usage: ${APP_NAME} mcp ${command} [-l]\n${HELP_HINT}`); return 1; } const entry = loaded.servers.find((server) => server.name === name); if (!entry) { + const names = loaded.servers.map((server) => server.name); + const suggestion = names.find((candidate) => editDistance(candidate, name) <= 2); error( - `No MCP server named "${name}".${untrustedNote ? ` ${untrustedNote}` : ""} Configured: ${loaded.servers.map((server) => server.name).join(", ") || "none"}.`, + `No MCP server named "${name}".${suggestion ? ` Did you mean "${suggestion}"?` : ""}${untrustedNote ? ` ${untrustedNote}` : ""} Configured: ${names.join(", ") || "none"}.`, ); return 1; } + if (parsed.values.has("local") && entry.scope !== "project") { + error(`MCP server "${name}" is defined in ${entry.source}, not in the project; omit --local.`); + return 1; + } const connection = createConnection(entry, options, credentials); const url = connection.oauthUrl; if (!url) { @@ -259,6 +272,19 @@ export async function runMcpCommand(args: string[], options: McpCommandOptions): } } +/** Levenshtein distance, to suggest a configured server name for a mistyped one. */ +function editDistance(a: string, b: string): number { + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + for (let j = 1; j <= b.length; j++) { + current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1)); + } + previous = current; + } + return previous[b.length]; +} + /** Parse `KEY=VALUE` pairs of a repeatable option into a record. */ function parsePairs(option: string, pairs: string[] | undefined, error: (line: string) => void) { if (!pairs) return {}; @@ -295,6 +321,7 @@ function add( "oauth-client-secret": "value", "oauth-callback-port": "value", "oauth-client-name": "value", + "oauth-client-registration": "value", exposure: "value", description: "value", }, @@ -321,6 +348,7 @@ function add( "oauth-client-secret", "oauth-callback-port", "oauth-client-name", + "oauth-client-registration", ]; const stdioOnly = ["env", "cwd"]; const misplaced = (url === undefined ? httpOnly : stdioOnly).find( @@ -343,6 +371,9 @@ function add( ...(value("oauth-client-secret") === undefined ? {} : { clientSecret: value("oauth-client-secret") }), ...(port === undefined ? {} : { callbackPort: Number(port) }), ...(value("oauth-client-name") === undefined ? {} : { clientName: value("oauth-client-name") }), + ...(value("oauth-client-registration") === undefined + ? {} + : { clientRegistration: value("oauth-client-registration") }), }; config = { url, diff --git a/packages/coding-agent/src/extensions/mcp/oauth.ts b/packages/coding-agent/src/extensions/mcp/oauth.ts index b1ca5601a..97fb56793 100644 --- a/packages/coding-agent/src/extensions/mcp/oauth.ts +++ b/packages/coding-agent/src/extensions/mcp/oauth.ts @@ -34,7 +34,7 @@ import { import lockfile from "proper-lockfile"; import { APP_NAME, getAgentDir } from "../../config.ts"; import { type AuthStorageBackend, FileAuthStorageBackend } from "../../core/auth-storage.ts"; -import { mcpNamespace } from "../../core/mcp-servers.ts"; +import { isClientMetadataDocumentCallback, mcpNamespace } from "../../core/mcp-servers.ts"; const CALLBACK_HOST = "127.0.0.1"; const CALLBACK_PATH = "/callback"; @@ -67,7 +67,7 @@ export interface McpOAuthSettings { /** `client_name` for dynamic client registration. Default: `APP_NAME`. */ clientName?: string; /** See `McpOAuthConfig.clientRegistration`. */ - clientRegistration?: "dcr" | "cimd"; + clientRegistration?: "auto" | "dcr" | "cimd"; /** See `McpOAuthConfig.authServerMetadataUrl`. */ authServerMetadataUrl?: URL; } @@ -243,23 +243,33 @@ function callbackId(serverUrl: string): string { } /** - * pi's Client ID Metadata Document, for `clientRegistration: "cimd"`, chosen like Codex chooses its own. - * The configuration ensures the default callback path. Without the `iss` parameter in authorization - * responses (RFC 9207), the redirect URI and the document are specific to the MCP server, so a response - * cannot be mixed up with one from another authorization server (RFC 9700 section 4.4.2.2). + * Whether sign-in may identify with pi's Client ID Metadata Document instead of registering. `auto` + * registers when the user asked for a client name, the redirect URI is not one the documents list, or + * a rebranded build would show up as pi. + */ +function offersClientMetadataDocument(settings: McpOAuthSettings): boolean { + const registration = settings.clientRegistration ?? "auto"; + if (registration !== "auto") return registration === "cimd"; + if (settings.clientId || settings.clientName || APP_NAME !== "pi") return false; + return isClientMetadataDocumentCallback(settings.callbackUrl); +} + +/** + * pi's Client ID Metadata Document, chosen like Codex chooses its own, or `undefined` when the + * authorization server does not support documents for public clients. Without the `iss` parameter in + * authorization responses (RFC 9207), the redirect URI and the document are specific to the MCP server, + * so a response cannot be mixed up with one from another authorization server (RFC 9700 section 4.4.2.2). */ function clientMetadataDocument( serverUrl: string, redirectUrl: string, metadata: AuthorizationServerMetadata | undefined, -): OAuthClientMetadataDocument { +): OAuthClientMetadataDocument | undefined { if ( !metadata?.client_id_metadata_document_supported || !metadata.token_endpoint_auth_methods_supported?.includes("none") ) { - throw new Error( - 'The authorization server does not support Client ID Metadata Documents for public clients; remove oauth.clientRegistration "cimd"', - ); + return undefined; } if (metadata.authorization_response_iss_parameter_supported) { return { url: `${CLIENT_METADATA_BASE_URL}/client.json`, redirectUrl }; @@ -281,10 +291,17 @@ function createProvider( serverUrl, redirectUrl, clientMetadata: { client_name: settings.clientName ?? APP_NAME }, - clientMetadataDocument: - settings.clientRegistration === "cimd" - ? (metadata) => clientMetadataDocument(serverUrl, redirectUrl, metadata) - : undefined, + clientMetadataDocument: offersClientMetadataDocument(settings) + ? (metadata) => { + const document = clientMetadataDocument(serverUrl, redirectUrl, metadata); + if (!document && settings.clientRegistration === "cimd") { + throw new Error( + 'The authorization server does not support Client ID Metadata Documents for public clients; remove oauth.clientRegistration "cimd"', + ); + } + return document; + } + : undefined, clientId: settings.clientId, clientSecret: settings.clientSecret, store, @@ -487,11 +504,11 @@ export async function signInMcpServer(options: { const registered = registeredRedirectUrls(stored?.clientInformation)[0]; const preferredPort = callbackOptions.port ?? (registered ? Number(new URL(registered).port) || undefined : undefined); - const cimd = settings.clientRegistration === "cimd"; + const offersDocument = offersClientMetadataDocument(settings); const callback = await listenForCallback( callbackOptions, // The redirect URI of a server-specific Client ID Metadata Document. - cimd ? [`${CALLBACK_PATH}/${callbackId(serverUrl)}`] : [], + offersDocument ? [`${CALLBACK_PATH}/${callbackId(serverUrl)}`] : [], preferredPort, callbackOptions.port !== undefined, ); @@ -501,11 +518,15 @@ export async function signInMcpServer(options: { const next: McpOAuthState = { ...stored }; // Every sign-in gets a fresh `state` parameter. delete next.oauthState; - // A registered client cannot use another redirect URI, and its tokens belong to it. A Client ID - // Metadata Document is not stored, so with one, a stored client was registered before and is replaced. + // A registered client cannot use another redirect URI, and its tokens belong to it. `cimd` replaces a + // registered client; `auto` keeps it. A Client ID Metadata Document is not stored, so tokens without + // a stored client belong to the document. const keepClient = settings.clientId || - (cimd ? !stored.clientInformation : registeredRedirectUrls(stored.clientInformation).includes(redirectUrl)); + (stored.clientInformation + ? settings.clientRegistration !== "cimd" && + registeredRedirectUrls(stored.clientInformation).includes(redirectUrl) + : offersDocument); if (!keepClient) { delete next.clientInformation; delete next.tokens; diff --git a/packages/coding-agent/src/extensions/tool-search/tool.ts b/packages/coding-agent/src/extensions/tool-search/tool.ts index b9fa4ade0..2cf07c769 100644 --- a/packages/coding-agent/src/extensions/tool-search/tool.ts +++ b/packages/coding-agent/src/extensions/tool-search/tool.ts @@ -20,10 +20,13 @@ import type { export const TOOL_SEARCH_TOOL_NAME = "tool_search"; export const DEFAULT_TOOL_SEARCH_LIMIT = 8; -/** A tool as the ranker sees it: its name and the text built by {@link createToolSearchDocument}. */ +/** A tool as the ranker sees it: its name and the texts built by {@link createToolSearchDocument}. */ export interface ToolSearchDocument { name: string; + /** Description, schema descriptions, and property names. */ text: string; + /** Name, description, and instructions of the tool's namespace. */ + namespaceText?: string; } export interface ToolSearchMatch { @@ -36,6 +39,8 @@ export interface ToolRanker { rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[]; } +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); + const STOP_WORDS: ReadonlySet = new Set([ "a", "an", @@ -68,15 +73,22 @@ function stem(term: string): string { return term; } -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ +/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */ export function tokenize(text: string): string[] { - return text + const normalized = text + .normalize("NFC") .replace(/([a-z0-9])([A-Z])/g, "$1 $2") .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other + // text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); } function isObject(value: unknown): value is Record { @@ -101,18 +113,19 @@ function schemaText(schema: unknown, parts: string[]): void { } /** - * Search text of a tool: the name, the name with `_` - * as spaces, the description, schema descriptions and property names, and the namespace with its - * description and instructions. + * Search text of a tool: the description and the schema descriptions and property names, and + * separately the namespace name, description, and instructions. The ranker reads the name from `name`. */ export function createToolSearchDocument( tool: Pick, namespace?: ToolNamespace, ): ToolSearchDocument { - const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description]; + const parts = [tool.description]; schemaText(tool.parameters, parts); - if (namespace) parts.push(namespace.name, namespace.description ?? "", namespace.instructions ?? ""); - return { name: tool.name, text: parts.filter((part) => part.trim()).join(" ") }; + const namespaceText = namespace + ? [namespace.name, namespace.description ?? "", namespace.instructions ?? ""].join(" ") + : ""; + return { name: tool.name, text: parts.filter((part) => part.trim()).join(" "), namespaceText }; } /** Lowercase with characters invalid in script identifiers as `_`, so `dev-radius` matches `dev_radius`. */ @@ -124,18 +137,66 @@ function normalizeName(name: string): string { } /** - * Whether `query` names the tool: its full name or a suffix after a `__` separator, so `thread_read` - * and `app__thread_read` both name `mcp__app__thread_read`. + * Whether the whole query is the tool's name: its full name or a suffix after a `__` separator, so + * `thread_read` and `app__thread_read` both name `mcp__app__thread_read`. */ -function isNameQuery(query: string, name: string): boolean { +function isExactNameQuery(query: string, name: string): boolean { const normalized = normalizeName(name); return normalized === query || normalized.endsWith(`__${query}`); } +/** A document split into the fields the ranker weighs differently. */ +interface IndexedDocument { + name: string; + /** Terms of the name, for example `mcp`, `app`, `thread`, `read` for `mcp__app__thread_read`. */ + nameTerms: ReadonlySet; + /** Terms of the last `__` segment of the name, the part models say: `thread`, `read`. */ + localNameTerms: readonly string[]; + textCounts: ReadonlyMap; + textLength: number; + namespaceTerms: ReadonlySet; +} + +function indexDocument(document: ToolSearchDocument): IndexedDocument { + const textCounts = new Map(); + for (const term of tokenize(document.text)) textCounts.set(term, (textCounts.get(term) ?? 0) + 1); + const segments = document.name.split("__"); + return { + name: document.name, + nameTerms: new Set(tokenize(document.name)), + localNameTerms: [...new Set(tokenize(segments[segments.length - 1]))], + textCounts, + textLength: [...textCounts.values()].reduce((sum, count) => sum + count, 0), + namespaceTerms: new Set(tokenize(document.namespaceText ?? "")), + }; +} + /** - * Okapi BM25 with the usual parameters. A query that names a tool ranks that tool first, because - * length normalization can otherwise rank a short description sharing the name's words above it. - * Ties keep document order. + * A query term in the name counts this many times its IDF. The text part of a term is at most + * `k1 + 1` (2.2) times its IDF and about 1 for a single mention, so a name match beats any number of + * mentions in the description. + */ +const NAME_WEIGHT = 3; + +/** + * A query term in the namespace name, description, or instructions counts this many times its IDF. They + * are shared by every tool of the server, so they help find the server but not the tool in it. + */ +const NAMESPACE_WEIGHT = 0.5; + +/** + * Okapi BM25 over the description, with the name and the namespace as separate fields: + * - a tool that matches more of the query's terms, in any field, ranks above one that matches + * fewer, whatever their scores: for `screenshot window`, a tool whose description has both words + * beats `window`, whose name has one; + * - a query term in the name scores more than any number of mentions in the description; + * - a tool whose whole local name is in the query (`app` in `peekaboo app` for + * `mcp__peekaboo__app`) gets a bonus worth one more match of every query term, so among tools + * matching as many terms it mostly wins over tools that only mention the words; + * - a query that is exactly a tool's name ranks that tool first, before coverage. + * + * The description alone is length-normalized, so a long name or long namespace instructions do + * not lower a tool's score. Ties keep document order. */ export class Bm25Ranker implements ToolRanker { private readonly k1: number; @@ -147,35 +208,42 @@ export class Bm25Ranker implements ToolRanker { } rank(query: string, documents: readonly ToolSearchDocument[], limit: number): ToolSearchMatch[] { - const queryTerms = [...new Set(tokenize(query))]; - if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return []; - const termCounts = documents.map((document) => { - const counts = new Map(); - for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1); - return counts; - }); - const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0)); - const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1; + const queryTerms = new Set(tokenize(query)); + if (queryTerms.size === 0 || documents.length === 0 || limit <= 0) return []; + const indexed = documents.map(indexDocument); + const averageLength = indexed.reduce((sum, document) => sum + document.textLength, 0) / indexed.length || 1; const idf = new Map( - queryTerms.map((term) => { - const frequency = termCounts.filter((counts) => counts.has(term)).length; - return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; + [...queryTerms].map((term) => { + const frequency = indexed.filter( + (document) => + document.nameTerms.has(term) || document.textCounts.has(term) || document.namespaceTerms.has(term), + ).length; + return [term, Math.log(1 + (indexed.length - frequency + 0.5) / (frequency + 0.5))] as const; }), ); - const nameQuery = normalizeName(query); - const matches: (ToolSearchMatch & { named: boolean })[] = []; - documents.forEach((document, index) => { + const fullNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0); + const exactQuery = normalizeName(query); + const matches: (ToolSearchMatch & { covered: number; exact: boolean })[] = []; + for (const document of indexed) { + const norm = this.k1 * (1 - this.b + (this.b * document.textLength) / averageLength); let score = 0; + let covered = 0; for (const term of queryTerms) { - const count = termCounts[index].get(term); - if (!count) continue; - const norm = this.k1 * (1 - this.b + (this.b * lengths[index]) / averageLength); - score += (idf.get(term) ?? 0) * ((count * (this.k1 + 1)) / (count + norm)); + const count = document.textCounts.get(term) ?? 0; + const weight = + (document.nameTerms.has(term) ? NAME_WEIGHT : 0) + + (document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) + + (count * (this.k1 + 1)) / (count + norm); + if (weight > 0) covered++; + score += (idf.get(term) ?? 0) * weight; } - if (score > 0) matches.push({ name: document.name, score, named: isNameQuery(nameQuery, document.name) }); - }); + if (score === 0) continue; + const local = document.localNameTerms; + if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += fullNameBonus; + matches.push({ name: document.name, score, covered, exact: isExactNameQuery(exactQuery, document.name) }); + } return matches - .sort((a, b) => Number(b.named) - Number(a.named) || b.score - a.score) + .sort((a, b) => Number(b.exact) - Number(a.exact) || b.covered - a.covered || b.score - a.score) .slice(0, limit) .map(({ name, score }) => ({ name, score })); } diff --git a/packages/coding-agent/src/index.ts b/packages/coding-agent/src/index.ts index f86470ad2..7e25c2cb8 100644 --- a/packages/coding-agent/src/index.ts +++ b/packages/coding-agent/src/index.ts @@ -118,6 +118,7 @@ export type { InputEvent, InputEventResult, InputSource, + KeybindingHints, KeybindingsManager, LoadExtensionsResult, LsToolCallEvent, diff --git a/packages/coding-agent/src/main.ts b/packages/coding-agent/src/main.ts index faba51f48..88aa3c5d3 100644 --- a/packages/coding-agent/src/main.ts +++ b/packages/coding-agent/src/main.ts @@ -756,9 +756,7 @@ export async function main(args: string[], options?: MainOptions) { parsed.projectTrustOverride === undefined && cachedProjectTrust === undefined && hasTrustRequiringResources; const projectTrusted = shouldResolveProjectTrust ? false - : (cachedProjectTrust ?? - parsed.projectTrustOverride ?? - (!hasTrustRequiringResources || trustStore.get(cwd) === true)); + : (cachedProjectTrust ?? parsed.projectTrustOverride ?? trustStore.get(cwd) ?? !hasTrustRequiringResources); const runtimeSettingsManager = SettingsManager.create(cwd, agentDir, { projectTrusted }); const services = await createAgentSessionServices({ cwd, diff --git a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts index 67bb71a3f..7cb296031 100644 --- a/packages/coding-agent/src/modes/interactive/components/assistant-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/assistant-message.ts @@ -16,6 +16,15 @@ const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; const OSC133_ZONE_FINAL = "\x1b]133;C\x07"; +/** + * A Markdown block of the last update. The next update reuses it at the same position for the same kind of content, so + * a streamed answer keeps the blocks it already rendered, also once the stream ends. + */ +interface MarkdownSlot { + kind: "text" | "thinking"; + markdown: Markdown; +} + /** * Component that renders a complete assistant message */ @@ -32,6 +41,7 @@ export class AssistantMessageComponent extends Container { private messageId?: string; private timestamp?: string; private thinkingVisibilityOverrides = new Map(); + private markdownSlots: MarkdownSlot[] = []; constructor( message?: AssistantMessage, @@ -86,6 +96,8 @@ export class AssistantMessageComponent extends Container { setOutputPad(padding: number): void { this.outputPad = padding; + // Markdown takes its padding at construction. + this.markdownSlots = []; if (this.lastMessage) { this.updateContent(this.lastMessage); } @@ -108,6 +120,15 @@ export class AssistantMessageComponent extends Container { // Clear content container this.contentContainer.clear(); + const previousSlots = this.markdownSlots; + this.markdownSlots = []; + const markdownBlock = (kind: MarkdownSlot["kind"], text: string, create: () => Markdown): Markdown => { + const slot = previousSlots[this.markdownSlots.length]; + const markdown = slot?.kind === kind ? slot.markdown : create(); + markdown.setText(text); + this.markdownSlots.push({ kind, markdown }); + return markdown; + }; // Spacers are added lazily, right before the next visible content block. This puts a blank // line before the first visible block and after each visible thinking run that is followed @@ -128,16 +149,22 @@ export class AssistantMessageComponent extends Container { if (content.type === "text" && content.text.trim()) { // Assistant text messages with no background - trim the text // Set paddingY=0 to avoid extra spacing before tool executions + const text = content.text.trim(); addVisibleChild( - new Markdown(content.text.trim(), this.outputPad, 0, this.markdownTheme, undefined, { - transform: createMarkdownTransform( - "assistant", - this.isStreaming, - this.markdownTransformers, - this.messageId, - this.timestamp, - ), - }), + markdownBlock( + "text", + text, + () => + new Markdown(text, this.outputPad, 0, this.markdownTheme, undefined, { + transform: createMarkdownTransform( + "assistant", + () => this.isStreaming, + this.markdownTransformers, + this.messageId, + this.timestamp, + ), + }), + ), ); } else if (content.type === "thinking") { const thinkingBlocks: string[] = []; @@ -163,26 +190,32 @@ export class AssistantMessageComponent extends Container { if (hidden && !this.hiddenThinkingLabel.trim()) { continue; } + const thinking = thinkingBlocks.join("\n\n"); const thinkingComponent = hidden ? new Text(theme.italic(theme.fg("thinkingText", this.hiddenThinkingLabel)), this.outputPad, 0) - : new Markdown( - thinkingBlocks.join("\n\n"), - this.outputPad, - 0, - this.markdownTheme, - { - color: (text: string) => theme.fg("thinkingText", text), - italic: true, - }, - { - transform: createMarkdownTransform( - "assistant-thinking", - this.isStreaming, - this.markdownTransformers, - this.messageId, - this.timestamp, + : markdownBlock( + "thinking", + thinking, + () => + new Markdown( + thinking, + this.outputPad, + 0, + this.markdownTheme, + { + color: (text: string) => theme.fg("thinkingText", text), + italic: true, + }, + { + transform: createMarkdownTransform( + "assistant-thinking", + () => this.isStreaming, + this.markdownTransformers, + this.messageId, + this.timestamp, + ), + }, ), - }, ); addVisibleChild( new MouseRegion(thinkingComponent, (event) => { diff --git a/packages/coding-agent/src/modes/interactive/components/config-selector.ts b/packages/coding-agent/src/modes/interactive/components/config-selector.ts index a8054886e..913b51179 100644 --- a/packages/coding-agent/src/modes/interactive/components/config-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/config-selector.ts @@ -2,7 +2,6 @@ * TUI component for managing package resources (enable/disable) */ -import { homedir } from "node:os"; import { basename, dirname, join, relative } from "node:path"; import { type Component, @@ -18,8 +17,8 @@ import { import { CONFIG_DIR_NAME } from "../../../config.ts"; import type { PathMetadata, ResolvedPaths, ResolvedResource } from "../../../core/package-manager.ts"; import type { PackageSource, SettingsManager } from "../../../core/settings-manager.ts"; -import { BUILTIN_PATH_PREFIX } from "../../../core/source-info.ts"; -import { canonicalizePath, isLocalPath, resolvePath } from "../../../utils/paths.ts"; +import { BUILTIN_PATH_PREFIX, getResourcePathKey } from "../../../core/source-info.ts"; +import { isLocalPath, resolvePath, shortenHomePath } from "../../../utils/paths.ts"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { keyHint, rawKeyHint } from "./keybinding-hints.ts"; @@ -65,19 +64,7 @@ interface ResourceGroup { } function formatBaseDir(baseDir: string): string { - const homeDir = homedir(); - let displayPath: string; - - if (baseDir === homeDir) { - displayPath = "~"; - } else if (baseDir.startsWith(homeDir)) { - // Replace home prefix with ~, normalize separators for display - const rest = baseDir.slice(homeDir.length); - displayPath = `~${rest.replace(/\\/g, "/")}`; - } else { - displayPath = baseDir.replace(/\\/g, "/"); - } - + const displayPath = shortenHomePath(baseDir).replace(/\\/g, "/"); return displayPath.endsWith("/") ? displayPath : `${displayPath}/`; } @@ -848,7 +835,7 @@ class ResourceList implements Component, Focusable { } private getResourceItemKey(item: ResourceItem): string { - return `${item.resourceType}:${canonicalizePath(item.path)}`; + return `${item.resourceType}:${getResourcePathKey(item.path)}`; } private getItemScope(item: ResourceItem): SettingsScope { diff --git a/packages/coding-agent/src/modes/interactive/components/custom-entry.ts b/packages/coding-agent/src/modes/interactive/components/custom-entry.ts index 078ad9fca..81d381ee9 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-entry.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-entry.ts @@ -3,6 +3,7 @@ import { Box, Container, Spacer, Text } from "@earendil-works/pi-tui"; import type { EntryRenderer } from "../../../core/extensions/types.ts"; import type { CustomEntry } from "../../../core/session-manager.ts"; import { theme } from "../theme/theme.ts"; +import { keyHint, keyText } from "./keybinding-hints.ts"; /** * Component that renders a custom session entry from extensions. @@ -50,7 +51,7 @@ export class CustomEntryComponent extends Container { let component: Component | undefined; try { - component = this.renderer(this.entry, { expanded: this._expanded }, theme); + component = this.renderer(this.entry, { expanded: this._expanded, keyText, keyHint }, theme); } catch (error) { const message = error instanceof Error ? error.message : String(error); const box = new Box(this.outputPad, 1, (text) => theme.bg("customMessageBg", text)); diff --git a/packages/coding-agent/src/modes/interactive/components/custom-message.ts b/packages/coding-agent/src/modes/interactive/components/custom-message.ts index 6c757e0dd..6dcddf402 100644 --- a/packages/coding-agent/src/modes/interactive/components/custom-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/custom-message.ts @@ -4,6 +4,7 @@ import { Box, Container, Markdown, type MarkdownTheme, Spacer, Text } from "@ear import type { MessageRenderer } from "../../../core/extensions/types.ts"; import type { CustomMessage } from "../../../core/messages.ts"; import { getMarkdownTheme, theme } from "../theme/theme.ts"; +import { keyHint, keyText } from "./keybinding-hints.ts"; /** * Component that renders a custom message entry from extensions. @@ -70,7 +71,7 @@ export class CustomMessageComponent extends Container { try { const component = this.customRenderer( this.message, - { expanded: this._expanded, outputPad: this.outputPad }, + { expanded: this._expanded, outputPad: this.outputPad, keyText, keyHint }, theme, ); if (component) { diff --git a/packages/coding-agent/src/modes/interactive/components/extension-editor.ts b/packages/coding-agent/src/modes/interactive/components/extension-editor.ts index 110447938..44f4bb220 100644 --- a/packages/coding-agent/src/modes/interactive/components/extension-editor.ts +++ b/packages/coding-agent/src/modes/interactive/components/extension-editor.ts @@ -30,6 +30,8 @@ export class ExtensionEditorComponent extends Container implements Focusable { private tui: TUI; private keybindings: KeybindingsManager; private externalEditorCommand: string; + /** Shows external editor failures; renders nothing while empty. */ + private externalEditorStatus = new Text("", 1, 0); private _focused = false; get focused(): boolean { @@ -85,6 +87,7 @@ export class ExtensionEditorComponent extends Container implements Focusable { this.onSubmitCallback(text); }; this.addChild(this.editor); + this.addChild(this.externalEditorStatus); this.addChild(new Spacer(1)); @@ -130,6 +133,7 @@ export class ExtensionEditorComponent extends Container implements Focusable { command: this.externalEditorCommand, content, }); + this.externalEditorStatus.setText(result.status === "failed" ? theme.fg("warning", result.message) : ""); if (result.status === "complete") { this.editor.setText(result.content); } diff --git a/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts b/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts index 4ff7685c2..58bae9094 100644 --- a/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts +++ b/packages/coding-agent/src/modes/interactive/components/markdown-transform.ts @@ -1,8 +1,9 @@ import type { MarkdownTransformContext, MarkdownTransformer } from "../../../core/extensions/types.ts"; +/** Applies extension Markdown transformers. `isStreaming` is read per transform, so a reused component sees a stream end. */ export function createMarkdownTransform( messageType: MarkdownTransformContext["messageType"], - isStreaming: boolean, + isStreaming: () => boolean, transformers: readonly MarkdownTransformer[], messageId?: string, timestamp?: string, @@ -10,7 +11,7 @@ export function createMarkdownTransform( return (markdown, availableWidth) => applyMarkdownTransformers( markdown, - { messageType, isStreaming, availableWidth, messageId, timestamp }, + { messageType, isStreaming: isStreaming(), availableWidth, messageId, timestamp }, transformers, ); } diff --git a/packages/coding-agent/src/modes/interactive/components/session-selector.ts b/packages/coding-agent/src/modes/interactive/components/session-selector.ts index baa66e991..4d6d6c09f 100644 --- a/packages/coding-agent/src/modes/interactive/components/session-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/session-selector.ts @@ -1,7 +1,6 @@ import { spawnSync } from "node:child_process"; import { existsSync } from "node:fs"; import { unlink } from "node:fs/promises"; -import * as os from "node:os"; import { type Component, Container, @@ -15,7 +14,7 @@ import { } from "@earendil-works/pi-tui"; import { KeybindingsManager } from "../../../core/keybindings.ts"; import type { SessionInfo, SessionListProgress } from "../../../core/session-manager.ts"; -import { canonicalizePath as _canonicalizePath } from "../../../utils/paths.ts"; +import { canonicalizePath, getPathComparisonKey, shortenHomePath } from "../../../utils/paths.ts"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { keyHint, keyText } from "./keybinding-hints.ts"; @@ -23,15 +22,6 @@ import { filterAndSortSessions, hasSessionName, type NameFilter, type SortMode } type SessionScope = "current" | "all"; -function shortenPath(path: string): string { - const home = os.homedir(); - if (!path) return path; - if (path.startsWith(home)) { - return `~${path.slice(home.length)}`; - } - return path; -} - function formatSessionDate(date: Date): string { const now = new Date(); const diffMs = now.getTime() - date.getTime(); @@ -48,9 +38,9 @@ function formatSessionDate(date: Date): string { return `${Math.floor(diffDays / 365)}y`; } -function canonicalizePath(path: string | undefined): string | undefined { - if (!path) return path; - return _canonicalizePath(path); +/** Comparison key for a session file path; follows symlinks and ignores case on Windows. */ +function getSessionPathKey(path: string): string { + return getPathComparisonKey(canonicalizePath(path)); } class SessionSelectorHeader implements Component { @@ -210,16 +200,16 @@ function buildSessionTree(sessions: SessionInfo[]): SessionTreeNode[] { const byPath = new Map(); for (const session of sessions) { - const sessionPath = canonicalizePath(session.path) ?? session.path; + const sessionPath = getSessionPathKey(session.path); byPath.set(sessionPath, { session, children: [], latestActivity: session.modified.getTime() }); } const roots: SessionTreeNode[] = []; for (const session of sessions) { - const sessionPath = canonicalizePath(session.path) ?? session.path; + const sessionPath = getSessionPathKey(session.path); const node = byPath.get(sessionPath)!; - const parentPath = canonicalizePath(session.parentSessionPath); + const parentPath = session.parentSessionPath ? getSessionPathKey(session.parentSessionPath) : undefined; if (parentPath && byPath.has(parentPath)) { byPath.get(parentPath)!.children.push(node); @@ -296,7 +286,7 @@ class SessionList implements Component, Focusable { private keybindings: KeybindingsManager; private showPath = false; private confirmingDeletePath: string | null = null; - private currentSessionCanonicalPath?: string; + private currentSessionPathKey?: string; public onSelect?: (sessionPath: string) => void; public onCancel?: () => void; public onExit: () => void = () => {}; @@ -335,7 +325,7 @@ class SessionList implements Component, Focusable { this.sortMode = sortMode; this.nameFilter = nameFilter; this.keybindings = keybindings; - this.currentSessionCanonicalPath = canonicalizePath(currentSessionFilePath); + this.currentSessionPathKey = currentSessionFilePath ? getSessionPathKey(currentSessionFilePath) : undefined; this.filterSessions(""); // Handle Enter in search input - select current item @@ -413,8 +403,8 @@ class SessionList implements Component, Focusable { } private isCurrentSessionPath(path: string): boolean { - if (!this.currentSessionCanonicalPath) return false; - return (canonicalizePath(path) ?? path) === this.currentSessionCanonicalPath; + if (!this.currentSessionPathKey) return false; + return getSessionPathKey(path) === this.currentSessionPathKey; } invalidate(): void {} @@ -474,10 +464,10 @@ class SessionList implements Component, Focusable { const msgCount = String(session.messageCount); let rightPart = `${msgCount} ${age}`; if (this.showCwd && session.cwd) { - rightPart = `${shortenPath(session.cwd)} ${rightPart}`; + rightPart = `${shortenHomePath(session.cwd)} ${rightPart}`; } if (this.showPath) { - rightPart = `${shortenPath(session.path)} ${rightPart}`; + rightPart = `${shortenHomePath(session.path)} ${rightPart}`; } // Cursor diff --git a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts index e739fc503..6d691b426 100644 --- a/packages/coding-agent/src/modes/interactive/components/tool-execution.ts +++ b/packages/coding-agent/src/modes/interactive/components/tool-execution.ts @@ -18,7 +18,7 @@ export type { ToolRenderers }; import { formatToolCallWithArgs, getTextOutput as getRenderedTextOutput } from "../../../core/tools/render-utils.ts"; import { ensurePngTranscoder } from "../../../utils/image-convert.ts"; import { theme } from "../theme/theme.ts"; -import { keyHint } from "./keybinding-hints.ts"; +import { keyHint, keyText } from "./keybinding-hints.ts"; const FALLBACK_PREVIEW_LINES = 10; @@ -138,6 +138,8 @@ export class ToolExecutionComponent extends Container { isError: this.result?.isError ?? false, durationMs: this.isPartial ? undefined : this.result?.durationMs, outputPad: this.outputPad, + keyText, + keyHint, }; } diff --git a/packages/coding-agent/src/modes/interactive/components/tree-selector.ts b/packages/coding-agent/src/modes/interactive/components/tree-selector.ts index 73255479b..5bcdd846d 100644 --- a/packages/coding-agent/src/modes/interactive/components/tree-selector.ts +++ b/packages/coding-agent/src/modes/interactive/components/tree-selector.ts @@ -14,6 +14,7 @@ import { } from "@earendil-works/pi-tui"; import type { SessionTreeNode } from "../../../core/session-manager.ts"; import { formatReadLineRange } from "../../../core/tools/render-utils.ts"; +import { shortenHomePath } from "../../../utils/paths.ts"; import { theme } from "../theme/theme.ts"; import { DynamicBorder } from "./dynamic-border.ts"; import { formatKeyText, keyHint } from "./keybinding-hints.ts"; @@ -945,11 +946,7 @@ class TreeList implements Component { } private formatToolCall(name: string, args: Record): string { - const shortenPath = (p: string): string => { - const home = process.env.HOME || process.env.USERPROFILE || ""; - if (home && p.startsWith(home)) return `~${p.slice(home.length)}`; - return p; - }; + const shortenPath = (p: string): string => shortenHomePath(p, process.env.HOME || process.env.USERPROFILE || ""); switch (name) { case "read": { diff --git a/packages/coding-agent/src/modes/interactive/components/user-message.ts b/packages/coding-agent/src/modes/interactive/components/user-message.ts index d8ca373cc..e313e0242 100644 --- a/packages/coding-agent/src/modes/interactive/components/user-message.ts +++ b/packages/coding-agent/src/modes/interactive/components/user-message.ts @@ -60,7 +60,7 @@ export class UserMessageComponent extends Container { preserveBackslashEscapes: true, transform: createMarkdownTransform( "user", - false, + () => false, this.markdownTransformers, this.messageId, this.timestamp, diff --git a/packages/coding-agent/src/modes/interactive/external-editor.ts b/packages/coding-agent/src/modes/interactive/external-editor.ts index d672d4e08..c4a0def9f 100644 --- a/packages/coding-agent/src/modes/interactive/external-editor.ts +++ b/packages/coding-agent/src/modes/interactive/external-editor.ts @@ -9,7 +9,7 @@ export interface ExternalEditorOptions { content: string; } -export type ExternalEditorResult = { status: "complete"; content: string } | { status: "failed" }; +export type ExternalEditorResult = { status: "complete"; content: string } | { status: "failed"; message: string }; export async function editInExternalEditor(options: ExternalEditorOptions): Promise { const directory = mkdtempSync(join(tmpdir(), "pi-editor-")); @@ -22,17 +22,29 @@ export async function editInExternalEditor(options: ExternalEditorOptions): Prom // Do not use spawnSync here. On Windows, synchronous child_process calls can keep // Node/libuv's console input read active after the parent pauses stdin, racing // vim/nvim for the console input buffer until Ctrl+C cancels the pending read. - const exitCode = await new Promise((resolve) => { + // Null means the editor could not be launched. + const exit = await new Promise((resolve) => { + // The editor inherits pi's cwd so project-relative paths, integrated terminals, + // and similar editor features keep working. A long cwd can break Windows editors: + // Windows rejects working directories over MAX_PATH, and WSL launches Windows + // editors (code, notepad.exe) from the cwd's \\wsl.localhost\... path (#11592). + // The cwd is intentionally not changed; failures are reported to the user instead. const child = spawn(editor, [...editorArgs, filePath], { stdio: "inherit", shell: process.platform === "win32", }); child.on("error", () => resolve(null)); - child.on("close", (code) => resolve(code)); + child.on("close", (code, signal) => resolve(code ?? signal)); }); - if (exitCode !== 0) { - return { status: "failed" }; + if (exit === null) { + return { status: "failed", message: `Failed to launch external editor: ${options.command}` }; + } + if (exit !== 0) { + return { + status: "failed", + message: `External editor exited with ${typeof exit === "number" ? `code ${exit}` : exit}; prompt unchanged`, + }; } return { status: "complete", content: stripBom(readFileSync(filePath, "utf-8")).replace(/\n$/, "") }; diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts index 9707bf5dc..6c6fbcf37 100644 --- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts +++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts @@ -127,7 +127,7 @@ import { copyToClipboard, readClipboardFilePaths, readClipboardText } from "../. import { extensionForImageMimeType, readClipboardImage } from "../../utils/clipboard-image.ts"; import { parseGitUrl } from "../../utils/git.ts"; import { ensurePngTranscoder } from "../../utils/image-convert.ts"; -import { getCwdRelativePath } from "../../utils/paths.ts"; +import { getCwdRelativePath, shortenHomePath } from "../../utils/paths.ts"; import { getPiUserAgent } from "../../utils/pi-user-agent.ts"; import { killTrackedDetachedChildren } from "../../utils/shell.ts"; import { loadAllHighlightLanguages } from "../../utils/syntax-highlight.ts"; @@ -1474,15 +1474,7 @@ export class InteractiveMode { // ========================================================================= private formatDisplayPath(p: string): string { - const home = os.homedir(); - let result = p; - - // Replace home directory with ~ - if (result.startsWith(home)) { - result = `~${result.slice(home.length)}`; - } - - return result; + return shortenHomePath(p); } private formatExtensionDisplayPath(path: string): string { @@ -2025,7 +2017,7 @@ export class InteractiveMode { if (skillDiagnostics.length > 0) { const warningLines = () => this.formatDiagnostics(skillDiagnostics, sourceInfos); this.loadedResourcesContainer.addChild( - new ThemedText(() => `${theme.fg("warning", "[Skill conflicts]")}\n${warningLines()}`, 0, 0), + new ThemedText(() => `${theme.fg("warning", "[Skill warnings]")}\n${warningLines()}`, 0, 0), ); this.loadedResourcesContainer.addChild(new Spacer(1)); } @@ -4708,6 +4700,8 @@ export class InteractiveMode { }); if (result.status === "complete") { this.editor.setText(result.content); + } else { + this.showStatus(result.message); } } finally { this.ui.start(); diff --git a/packages/coding-agent/src/modes/interactive/theme/theme.ts b/packages/coding-agent/src/modes/interactive/theme/theme.ts index 876ba9e86..922b23d9f 100644 --- a/packages/coding-agent/src/modes/interactive/theme/theme.ts +++ b/packages/coding-agent/src/modes/interactive/theme/theme.ts @@ -26,7 +26,7 @@ import chalk from "chalk"; import { getCustomThemesDir, getThemesDir } from "../../../config.ts"; import type { SourceInfo } from "../../../core/source-info.ts"; import { closeWatcher, watchWithErrorHandler } from "../../../utils/fs-watch.ts"; -import { highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts"; +import { getHighlightLanguageCount, highlight, supportsLanguage } from "../../../utils/syntax-highlight.ts"; import { stripBom } from "../../../utils/text.ts"; import { generateSystemThemeColors, SYSTEM_THEME_NAME, terminalAppearance } from "./system-theme.ts"; import { @@ -658,6 +658,11 @@ export const theme: Theme = new Proxy({} as Theme, { }, }); +/** The theme instance behind the `theme` proxy, whose own identity never changes. */ +function getGlobalTheme(): Theme | undefined { + return (globalThis as Record)[THEME_KEY]; +} + function setGlobalTheme(t: Theme): void { (globalThis as Record)[THEME_KEY] = t; (globalThis as Record)[THEME_KEY_OLD] = t; @@ -944,7 +949,41 @@ export function highlightCode(code: string, lang?: string): string[] { try { return highlight(code, opts).split("\n"); } catch { - return code.split("\n"); + return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); + } +} + +/** + * Highlights code for one owner, such as a tool row, and reuses the last result while the code, the language, the + * theme and the registered grammars stay the same. Each owner keeps its own highlighter, so cached results live as long + * as the code they show instead of in a process-wide cache. + */ +export class CodeHighlighter { + private last?: { + code: string; + lang: string | undefined; + theme: Theme | undefined; + languageCount: number; + lines: readonly string[]; + }; + + /** Like {@link highlightCode}. The result is shared with later calls, so callers must not modify it. */ + highlight(code: string, lang: string | undefined): readonly string[] { + // Renderers cannot tell a theme change from any other redraw, so the highlighter checks what its result depends on. + const currentTheme = getGlobalTheme(); + const languageCount = getHighlightLanguageCount(); + const last = this.last; + if ( + last?.code === code && + last.lang === lang && + last.theme === currentTheme && + last.languageCount === languageCount + ) { + return last.lines; + } + const lines = highlightCode(code, lang); + this.last = { code, lang, theme: currentTheme, languageCount, lines }; + return lines; } } @@ -1035,26 +1074,7 @@ export function getMarkdownTheme(): MarkdownTheme { italic: (text: string) => theme.italic(text), underline: (text: string) => theme.underline(text), strikethrough: (text: string) => theme.strikethrough(text), - highlightCode: (code: string, lang?: string): string[] => { - // Validate language before highlighting to avoid stderr spam from cli-highlight - const validLang = lang && supportsLanguage(lang) ? lang : undefined; - // Skip highlighting when no valid language is specified. cli-highlight's - // auto-detection is unreliable and can misidentify prose as AppleScript, - // LiveCodeServer, etc., coloring random English words as keywords. - if (!validLang) { - return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); - } - const opts = { - language: validLang, - ignoreIllegals: true, - theme: getCliHighlightTheme(theme), - }; - try { - return highlight(code, opts).split("\n"); - } catch { - return code.split("\n").map((line) => theme.fg("mdCodeBlock", line)); - } - }, + highlightCode, }; } diff --git a/packages/coding-agent/src/package-manager-cli.ts b/packages/coding-agent/src/package-manager-cli.ts index 5696b66ad..93a8fd4f8 100644 --- a/packages/coding-agent/src/package-manager-cli.ts +++ b/packages/coding-agent/src/package-manager-cli.ts @@ -17,11 +17,14 @@ import { createProjectTrustContext } from "./cli/project-trust.ts"; import { APP_NAME, CONFIG_DIR_NAME, + detectInstallChange, detectInstallMethod, getAgentDir, + getConfiguredSelfUpdateArgv, getPackageDir, getSelfUpdateCommand, getSelfUpdateUnavailableInstruction, + makeConfiguredSelfUpdateCommand, PACKAGE_NAME, type SelfUpdateCommand, type SelfUpdatePackageTarget, @@ -29,7 +32,7 @@ import { } from "./config.ts"; import type { InlineExtension } from "./core/extensions/types.ts"; import { ModelRuntime } from "./core/model-runtime.ts"; -import { DefaultPackageManager } from "./core/package-manager.ts"; +import { DefaultPackageManager, isOfflineModeEnabled } from "./core/package-manager.ts"; import { type AppMode, resolveProjectTrusted } from "./core/project-trust.ts"; import { DefaultResourceLoader, formatPackageInstallWarning, isBuiltinExtension } from "./core/resource-loader.ts"; import { SettingsManager } from "./core/settings-manager.ts"; @@ -638,6 +641,9 @@ function printSelfUpdateUnavailable( ): void { console.error(`error: ${APP_NAME} cannot self-update this installation.`); console.error(getSelfUpdateUnavailableInstruction(PACKAGE_NAME, npmCommand, updatePackageTarget)); + console.error( + `To make \`${APP_NAME} update\` run your own update command, set PI_SELF_UPDATE_COMMAND or the selfUpdateCommand setting.`, + ); const entrypoint = process.argv[1]; if (entrypoint) { @@ -735,6 +741,48 @@ async function runSelfUpdate(command: SelfUpdateCommand): Promise { } } +/** Run the user-configured self-update command, then report the version installed on disk. */ +async function runConfiguredSelfUpdate(argv: string[], plan: SelfUpdatePlan): Promise { + const command = makeConfiguredSelfUpdateCommand(argv, plan); + if (plan.note) { + printSelfUpdateNote(plan.note); + } + try { + await runSelfUpdate(command); + } catch (error: unknown) { + const message = error instanceof Error ? error.message : "Unknown self-update command error"; + console.error(chalk.red(`Error: ${message}`)); + process.exitCode = 1; + return; + } + // Exit code 0 does not prove an update. Wrappers such as mise or Nix install each version into a + // new directory and leave this installation untouched, so fall back to the pi on PATH. + const change = detectInstallChange(); + const installedVersion = + change?.kind === "updated" && change.version === plan.version ? change.version : readVersionOnPath(); + if (installedVersion === plan.version) { + console.log(chalk.green(`Updated ${APP_NAME} from ${VERSION} to ${plan.version}`)); + return; + } + console.log( + chalk.yellow( + `Ran ${command.display}, but \`${APP_NAME} --version\` reports ${installedVersion ?? "no version"}, not ${plan.version}.`, + ), + ); +} + +/** Version printed by `pi --version` as resolved from PATH, the pi the user runs next. */ +function readVersionOnPath(): string | undefined { + // PI_PACKAGE_DIR points at this installation and would make the new pi report the old version. + const { PI_PACKAGE_DIR: _, ...env } = process.env; + const result = spawnProcessSync(APP_NAME, ["--version"], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + env, + }); + return result.status === 0 ? result.stdout.trim() || undefined : undefined; +} + function prepareWindowsNpmSelfUpdate(): void { if (process.platform !== "win32") { return; @@ -979,9 +1027,18 @@ export async function handlePackageCommand( const packageManager = new DefaultPackageManager({ cwd, agentDir, settingsManager }); + // Update result events describe a changed package, e.g. "npm:foo 1.2.0 -> 1.3.0". + const updatedPackages: string[] = []; + // Progress messages are warnings, e.g. npm reinstalled the old version from stale registry metadata. + let warningCount = 0; packageManager.setProgressCallback((event) => { - if (event.type === "start") { + if (event.type === "result" && event.action === "update" && event.message) { + updatedPackages.push(event.message); + } else if (event.type === "start") { process.stdout.write(chalk.dim(`${event.message}\n`)); + } else if (event.type === "progress" && event.message) { + warningCount++; + process.stdout.write(chalk.yellow(`${event.message}\n`)); } }); @@ -1049,14 +1106,37 @@ export async function handlePackageCommand( if (updateTargetIncludesExtensions(target)) { const updateSource = target.type === "extensions" ? target.source : undefined; await packageManager.update(updateSource); - if (updateSource) { - console.log(chalk.green(`Updated ${updateSource}`)); + // Only claim "up to date" when the update actually checked and nothing warned. + if (isOfflineModeEnabled()) { + console.log(chalk.yellow("Skipped package updates (offline mode)")); + } else if (updatedPackages.length === 0 && warningCount > 0) { + console.log( + chalk.yellow( + updateSource + ? `${updateSource} was not updated, see the warnings above` + : "No packages were updated, see the warnings above", + ), + ); + } else if (updatedPackages.length === 0) { + console.log( + chalk.green( + updateSource ? `${updateSource} is already up to date` : "Packages are already up to date", + ), + ); } else { - console.log(chalk.green("Updated packages")); + const count = updatedPackages.length; + console.log(chalk.green(`Updated ${count} package${count === 1 ? "" : "s"}:`)); + for (const message of updatedPackages) { + console.log(` ${message}`); + } } } if (updateTargetIncludesSelf(target)) { - const managedInstallRoot = getActiveManagedInstallRoot(); + const configuredSelfUpdateArgv = getConfiguredSelfUpdateArgv( + settingsManager.getGlobalSettings().selfUpdateCommand, + ); + // A configured command is explicit user intent, so it replaces the managed install path too. + const managedInstallRoot = configuredSelfUpdateArgv ? undefined : getActiveManagedInstallRoot(); if (managedInstallRoot && options.force) { console.error( chalk.red( @@ -1070,6 +1150,10 @@ export async function handlePackageCommand( if (!selfUpdatePlan.shouldRun) { return true; } + if (configuredSelfUpdateArgv) { + await runConfiguredSelfUpdate(configuredSelfUpdateArgv, selfUpdatePlan); + return true; + } if (managedInstallRoot) { if (selfUpdatePlan.note) { printSelfUpdateNote(selfUpdatePlan.note); diff --git a/packages/coding-agent/src/utils/clipboard-command.ts b/packages/coding-agent/src/utils/clipboard-command.ts index 0cddbf59c..66601ab8c 100644 --- a/packages/coding-agent/src/utils/clipboard-command.ts +++ b/packages/coding-agent/src/utils/clipboard-command.ts @@ -10,8 +10,12 @@ export function runClipboardCommand( // Clipboard writers can daemonize. Do not give them output pipes to retain. // Detach on POSIX so the daemon leaves pi's process group and Ctrl+Z suspend // (SIGTSTP to the group) does not stop the clipboard owner (#11179). + // Run from / (the current drive root on Windows): clipboard tools ignore the cwd, and WSL cannot start + // Windows executables (powershell.exe) when the cwd's \\wsl.localhost\... path + // exceeds MAX_PATH (#11592). This also keeps daemons from pinning pi's cwd. const child = spawn(command, args, { stdio: ["pipe", options?.input === undefined ? "pipe" : "ignore", "ignore"], + cwd: "/", detached: process.platform !== "win32", windowsHide: true, }); diff --git a/packages/coding-agent/src/utils/git.ts b/packages/coding-agent/src/utils/git.ts index 1314edea7..2b5a1369d 100644 --- a/packages/coding-agent/src/utils/git.ts +++ b/packages/coding-agent/src/utils/git.ts @@ -14,7 +14,7 @@ export type GitSource = { path: string; /** Git ref (branch, tag, commit) if specified */ ref?: string; - /** True if ref was specified (package won't be auto-updated) */ + /** True if ref was specified. `pi update` reconciles the checkout to the ref but does not move the ref. */ pinned: boolean; }; diff --git a/packages/coding-agent/src/utils/highlight-js.d.ts b/packages/coding-agent/src/utils/highlight-js.d.ts index d4b71174e..c4c951fce 100644 --- a/packages/coding-agent/src/utils/highlight-js.d.ts +++ b/packages/coding-agent/src/utils/highlight-js.d.ts @@ -17,6 +17,7 @@ interface HighlightJsApi { highlight(code: string, options: HighlightJsOptions): HighlightJsResult; highlightAuto(code: string, languageSubset?: string[]): HighlightJsResult; getLanguage(name: string): HighlightJsLanguageDefinition | undefined; + listLanguages(): string[]; registerLanguage(name: string, language: HighlightJsLanguageFactory): void; } diff --git a/packages/coding-agent/src/utils/image-resize-core.ts b/packages/coding-agent/src/utils/image-resize-core.ts index 3f80111ff..d4a4d8cab 100644 --- a/packages/coding-agent/src/utils/image-resize-core.ts +++ b/packages/coding-agent/src/utils/image-resize-core.ts @@ -134,9 +134,11 @@ export async function resizeImageInProcess( let currentHeight = targetHeight; while (true) { - const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); - for (const candidate of candidates) { - if (candidate.encodedSize < opts.maxBytes) { + const [png, jpeg, ...lowerQualityJpegs] = tryEncodings(currentWidth, currentHeight, qualitySteps); + // Prefer the smaller of PNG and JPEG at the configured quality; lower JPEG quality only if neither fits. + const preferred = jpeg && jpeg.encodedSize < png.encodedSize ? [jpeg, png] : [png, jpeg]; + for (const candidate of [...preferred, ...lowerQualityJpegs]) { + if (candidate && candidate.encodedSize < opts.maxBytes) { return { data: candidate.data, mimeType: candidate.mimeType, diff --git a/packages/coding-agent/src/utils/open-browser.ts b/packages/coding-agent/src/utils/open-browser.ts index 435e23f97..727603a7d 100644 --- a/packages/coding-agent/src/utils/open-browser.ts +++ b/packages/coding-agent/src/utils/open-browser.ts @@ -18,7 +18,10 @@ export function openBrowser(target: string): void { // spawn reports launcher failures (for example, missing xdg-open) via an // error event. Browser launch is best-effort: callers still present the target // to the user, so keep the launcher failure from becoming a process crash. - spawn(cmd, args, { stdio: "ignore", detached: true }) + // On WSL, xdg-open launches explorer.exe, which fails when the cwd's + // \\wsl.localhost\... path exceeds MAX_PATH (#11592). The target is a URL, + // so no launcher needs pi's cwd. + spawn(cmd, args, { stdio: "ignore", detached: true, cwd: "/" }) .on("error", () => {}) .unref(); } diff --git a/packages/coding-agent/src/utils/paths.ts b/packages/coding-agent/src/utils/paths.ts index 6a5a0cfb8..cf77b5bff 100644 --- a/packages/coding-agent/src/utils/paths.ts +++ b/packages/coding-agent/src/utils/paths.ts @@ -1,6 +1,6 @@ import { realpathSync, statSync } from "node:fs"; import { homedir } from "node:os"; -import { isAbsolute, join, resolve as nodeResolvePath, relative, sep } from "node:path"; +import { isAbsolute, join, resolve as nodeResolvePath, posix, relative, sep, win32 } from "node:path"; import { fileURLToPath } from "node:url"; import { spawnProcessSync } from "./child-process.ts"; @@ -33,6 +33,50 @@ export function canonicalizePath(path: string): string { } } +/** + * Key for comparing filesystem paths: the resolved absolute path, case-folded + * on Windows because Windows paths are case-insensitive (`c:\Users` and + * `C:\Users` are the same directory). NTFS can enable per-directory case + * sensitivity, but it is rarely used outside WSL interop, so it is ignored + * here (https://learn.microsoft.com/en-us/windows/wsl/case-sensitivity). + * Does not touch the filesystem; call `canonicalizePath` first when symlinks + * must compare equal. Use only for comparison, never for display or + * filesystem access. + */ +export function getPathComparisonKey(path: string): string { + const pathApi = getPlatformPathApi(); + const resolvedPath = pathApi.resolve(path); + return pathApi === win32 ? resolvedPath.toLowerCase() : resolvedPath; +} + +/** `node:path` rules for the current platform, read at call time so tests can simulate Windows. */ +function getPlatformPathApi(): typeof win32 { + return process.platform === "win32" ? win32 : posix; +} + +/** True when both paths name the same location (see `getPathComparisonKey`). */ +export function pathsEqual(left: string, right: string): boolean { + return getPathComparisonKey(left) === getPathComparisonKey(right); +} + +/** True when `child` is `parent` or inside it (see `getPathComparisonKey`). */ +export function isPathWithin(parent: string, child: string): boolean { + const parentKey = getPathComparisonKey(parent); + const childKey = getPathComparisonKey(child); + if (childKey === parentKey) return true; + // Filesystem roots (`/`, `C:\`) already end with a separator. + const pathSep = getPlatformPathApi().sep; + return childKey.startsWith(parentKey.endsWith(pathSep) ? parentKey : `${parentKey}${pathSep}`); +} + +/** Replace a leading home directory with `~` for display. Relative paths are returned unchanged. */ +export function shortenHomePath(path: string, homeDir: string = homedir()): string { + const pathApi = getPlatformPathApi(); + if (!homeDir || !pathApi.isAbsolute(path) || !isPathWithin(homeDir, path)) return path; + const relativePath = pathApi.relative(homeDir, path); + return relativePath ? `~${pathApi.sep}${relativePath}` : "~"; +} + export function getFileRevision(path: string): string | undefined { try { const stats = statSync(path, { bigint: true }); diff --git a/packages/coding-agent/src/utils/syntax-highlight.ts b/packages/coding-agent/src/utils/syntax-highlight.ts index 1ee1986b9..91375bb01 100644 --- a/packages/coding-agent/src/utils/syntax-highlight.ts +++ b/packages/coding-agent/src/utils/syntax-highlight.ts @@ -217,3 +217,11 @@ export function highlight(code: string, options: HighlightOptions = {}): string export function supportsLanguage(name: string): boolean { return hljs.getLanguage(name) !== undefined; } + +/** + * Number of registered grammars. It grows when the deferred grammars load, which can change the highlighting of + * code that embeds another language, such as HTML in a JavaScript template literal. + */ +export function getHighlightLanguageCount(): number { + return hljs.listLanguages().length; +} diff --git a/packages/coding-agent/test/assistant-message.test.ts b/packages/coding-agent/test/assistant-message.test.ts index 42f6e496d..2cf368915 100644 --- a/packages/coding-agent/test/assistant-message.test.ts +++ b/packages/coding-agent/test/assistant-message.test.ts @@ -1,10 +1,11 @@ import type { AssistantMessage } from "@earendil-works/pi-ai"; import type { TuiMouseEvent } from "@earendil-works/pi-tui"; -import { describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { AssistantMessageComponent } from "../src/modes/interactive/components/assistant-message.ts"; import { UserMessageComponent } from "../src/modes/interactive/components/user-message.ts"; import { initTheme } from "../src/modes/interactive/theme/theme.ts"; import { stripAnsi } from "../src/utils/ansi.ts"; +import * as syntaxHighlight from "../src/utils/syntax-highlight.ts"; const OSC133_ZONE_START = "\x1b]133;A\x07"; const OSC133_ZONE_END = "\x1b]133;B\x07"; @@ -34,6 +35,37 @@ function createAssistantMessage( } describe("AssistantMessageComponent", () => { + afterEach(() => vi.restoreAllMocks()); + + test("reuses its Markdown across streaming and its end, and renders like a new component after a theme change", () => { + initTheme("dark"); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + const streaming: boolean[] = []; + const code = "```ts\nconst a = 1;\n```"; + const content = (text: string): AssistantMessage["content"] => [ + { type: "thinking", thinking: "Thinking with **bold** text" }, + { type: "text", text }, + ]; + const component = new AssistantMessageComponent(undefined, false, undefined, "Thinking...", 1, [ + (markdown, { isStreaming }) => { + streaming.push(isStreaming); + return markdown; + }, + ]); + component.updateContent(createAssistantMessage(content(code)), true); + component.render(80); + const message = createAssistantMessage(content(`${code}\n\nThe end.`)); + component.updateContent(message, false); + component.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + expect(streaming.at(-1)).toBe(false); + + initTheme("light"); + component.invalidate(); + expect(component.render(80)).toEqual(new AssistantMessageComponent(message).render(80)); + initTheme("dark"); + }); + test("adds OSC 133 zone markers to assistant messages without tool calls", () => { initTheme("dark"); diff --git a/packages/coding-agent/test/branch-summarization.test.ts b/packages/coding-agent/test/branch-summarization.test.ts index ade25dbfd..3164a8773 100644 --- a/packages/coding-agent/test/branch-summarization.test.ts +++ b/packages/coding-agent/test/branch-summarization.test.ts @@ -7,7 +7,7 @@ import { type SimpleStreamOptions, } from "@earendil-works/pi-ai"; import { describe, expect, it } from "vitest"; -import { generateBranchSummary } from "../src/core/compaction/index.ts"; +import { generateBranchSummary, prepareBranchEntries } from "../src/core/compaction/index.ts"; import type { SessionEntry } from "../src/core/session-manager.ts"; const model: Model<"anthropic-messages"> = { @@ -131,4 +131,54 @@ describe("branch summarization", () => { "Branch summarization failed: generation hit the token cap and the summary is incomplete", ); }); + + it("applies context edits to branch entries", () => { + const ts = (n: number) => new Date(n).toISOString(); + const branch: SessionEntry[] = [ + { + type: "message", + id: "u1", + parentId: null, + timestamp: ts(1), + message: { role: "user", content: "Investigate login", timestamp: 1 }, + }, + { + type: "message", + id: "failed", + parentId: "u1", + timestamp: ts(2), + message: fauxAssistantMessage("FAILED_ATTEMPT", { stopReason: "error", errorMessage: "lost" }), + }, + { + type: "context_edit", + id: "e1", + parentId: "failed", + timestamp: ts(3), + targetId: "failed", + replacement: null, + }, + { + type: "message", + id: "ok", + parentId: "e1", + timestamp: ts(4), + message: fauxAssistantMessage("ORIGINAL_TEXT"), + }, + { + type: "context_edit", + id: "e2", + parentId: "ok", + timestamp: ts(5), + targetId: "ok", + replacement: { content: "REPLACED_TEXT" }, + }, + ]; + + const serialized = JSON.stringify(prepareBranchEntries(branch).messages); + + expect(serialized).toContain("Investigate login"); + expect(serialized).not.toContain("FAILED_ATTEMPT"); + expect(serialized).not.toContain("ORIGINAL_TEXT"); + expect(serialized).toContain("REPLACED_TEXT"); + }); }); diff --git a/packages/coding-agent/test/clipboard-command.test.ts b/packages/coding-agent/test/clipboard-command.test.ts index 8e250f817..5be4220b0 100644 --- a/packages/coding-agent/test/clipboard-command.test.ts +++ b/packages/coding-agent/test/clipboard-command.test.ts @@ -1,3 +1,4 @@ +import { resolve } from "node:path"; import { describe, expect, test } from "vitest"; import { runClipboardCommand } from "../src/utils/clipboard-command.ts"; @@ -39,6 +40,11 @@ describe("clipboard commands", () => { expect(ownPgid).toMatch(/^\d+$/); expect(childPgid).not.toBe(ownPgid); }); + // #11592: WSL cannot start Windows executables from a cwd whose UNC path exceeds MAX_PATH. + test("runs commands from the filesystem root", async () => { + const cwd = await runClipboardCommand(process.execPath, ["-e", "process.stdout.write(process.cwd())"]); + expect(cwd?.toString("utf8")).toBe(resolve("/")); + }); test("rejects output above the buffer limit", async () => { expect( await runClipboardCommand(process.execPath, ["-e", "process.stdout.write(Buffer.alloc(1024))"], { diff --git a/packages/coding-agent/test/code-highlighter.test.ts b/packages/coding-agent/test/code-highlighter.test.ts new file mode 100644 index 000000000..b0d0433b4 --- /dev/null +++ b/packages/coding-agent/test/code-highlighter.test.ts @@ -0,0 +1,71 @@ +import type { TUI } from "@earendil-works/pi-tui"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { writeRenderers } from "../src/core/tools/renderers/write.ts"; +import { codemodeRenderers } from "../src/extensions/codemode/renderer.ts"; +import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; +import { CodeHighlighter, highlightCode, initTheme } from "../src/modes/interactive/theme/theme.ts"; +import * as syntaxHighlight from "../src/utils/syntax-highlight.ts"; + +const tui = { requestRender: () => {} } as unknown as TUI; + +describe("CodeHighlighter", () => { + beforeEach(() => initTheme("dark")); + afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it("reuses its last result until the code, the theme or the grammars change", async () => { + const code = 'const page = html`
ok
`;'; + const highlighter = new CodeHighlighter(); + const first = highlighter.highlight(code, "javascript"); + expect(highlighter.highlight(code, "javascript")).toBe(first); + expect(highlighter.highlight("const a = 1;", "javascript")).not.toBe(first); + + initTheme("light"); + expect(highlighter.highlight(code, "javascript")).toEqual(highlightCode(code, "javascript")); + const light = highlighter.highlight(code, "javascript"); + await syntaxHighlight.loadAllHighlightLanguages(); + expect(highlighter.highlight(code, "javascript")).not.toEqual(light); + }); + + it("codemode highlights the script once across progress, timer ticks, expanding and completion", () => { + vi.useFakeTimers(); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + const row = new ToolExecutionComponent( + "codemode", + "c", + { code: "await tools.read({});" }, + {}, + codemodeRenderers, + tui, + "/", + ); + row.setArgsComplete(); + row.markExecutionStarted(); + row.updateResult({ content: [], details: { calls: [] }, isError: false }, true); + vi.advanceTimersByTime(2_000); + row.setExpanded(true); + row.updateResult({ content: [{ type: "text", text: "done" }], details: { calls: [] }, isError: false }, false); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + }); + + it("write highlights unchanged first lines and a complete file only once", () => { + const content = Array.from({ length: 60 }, (_, i) => `const v${i} = ${i};`).join("\n"); + const row = new ToolExecutionComponent("write", "w", { path: "a.ts", content }, {}, writeRenderers, tui, "/"); + row.updateArgs({ path: "a.ts", content: `${content} //` }); + row.render(80); + const highlight = vi.spyOn(syntaxHighlight, "highlight"); + // Only the changed last line is highlighted again. + row.updateArgs({ path: "a.ts", content: `${content} // more` }); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(1); + + row.setArgsComplete(); + row.setExpanded(true); + row.setExpanded(false); + row.render(80); + expect(highlight).toHaveBeenCalledTimes(2); + }); +}); diff --git a/packages/coding-agent/test/codemode-renderer.test.ts b/packages/coding-agent/test/codemode-renderer.test.ts index 7c802da71..571933226 100644 --- a/packages/coding-agent/test/codemode-renderer.test.ts +++ b/packages/coding-agent/test/codemode-renderer.test.ts @@ -5,6 +5,7 @@ import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import type { ToolRenderContext } from "../src/core/extensions/types.ts"; import { codemodeRenderers } from "../src/extensions/codemode/renderer.ts"; import type { CodemodeToolDetails } from "../src/extensions/codemode/tool.ts"; +import { keyHint, keyText } from "../src/modes/interactive/components/keybinding-hints.ts"; import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; import { initTheme, theme } from "../src/modes/interactive/theme/theme.ts"; @@ -29,6 +30,8 @@ function render( isError, durationMs: undefined, outputPad: 1, + keyText, + keyHint, } satisfies ToolRenderContext; const component = codemodeRenderers.renderResult?.( result, diff --git a/packages/coding-agent/test/config.test.ts b/packages/coding-agent/test/config.test.ts index ff652cae3..ff2c8a733 100644 --- a/packages/coding-agent/test/config.test.ts +++ b/packages/coding-agent/test/config.test.ts @@ -6,9 +6,11 @@ import { detectInstallChange, detectInstallMethod, findNodePackageDir, + getConfiguredSelfUpdateArgv, getSelfUpdateCommand, getSelfUpdateUnavailableInstruction, getUpdateInstruction, + makeConfiguredSelfUpdateCommand, } from "../src/config.ts"; const execPathDescriptor = Object.getOwnPropertyDescriptor(process, "execPath"); @@ -182,6 +184,33 @@ describe("detectInstallMethod", () => { ); }); + // Issue #7756 + test("does not name pnpm for unmanaged installs under PNPM_HOME", () => { + tempDir = mkdtempSync(join(tmpdir(), "pi-pnpm-home-")); + const packageDir = join( + tempDir, + ".local", + "share", + "pnpm", + "global-nub", + "5", + "node_modules", + "@earendil-works", + "pi-coding-agent", + ); + mkdirSync(packageDir, { recursive: true }); + process.env.PI_PACKAGE_DIR = packageDir; + process.env.PATH = ""; + process.argv[1] = join(packageDir, "dist", "cli.js"); + setExecPath(join(packageDir, "dist", "cli.js")); + + expect(detectInstallMethod()).toBe("pnpm"); + expect(getSelfUpdateCommand("@earendil-works/pi-coding-agent")).toBeUndefined(); + expect(getSelfUpdateUnavailableInstruction("@earendil-works/pi-coding-agent")).toBe( + "Update @earendil-works/pi-coding-agent using the package manager, wrapper, or source checkout that provides this installation.", + ); + }); + test("self-updates npm installs from custom prefixes", () => { const { prefix } = createNpmPrefixInstall(); @@ -462,3 +491,54 @@ describe("detectInstallChange", () => { expect(detectInstallChange(join(installDir, "package.json"))).toEqual({ kind: "removed" }); }); }); + +describe("configured self-update command", () => { + const originalSelfUpdateCommand = process.env.PI_SELF_UPDATE_COMMAND; + + afterEach(() => { + if (originalSelfUpdateCommand === undefined) { + delete process.env.PI_SELF_UPDATE_COMMAND; + } else { + process.env.PI_SELF_UPDATE_COMMAND = originalSelfUpdateCommand; + } + }); + + test("prefers PI_SELF_UPDATE_COMMAND over the setting and splits it on whitespace", () => { + process.env.PI_SELF_UPDATE_COMMAND = " nub upgrade {installSpec} "; + + expect(getConfiguredSelfUpdateArgv(["mise", "upgrade"])).toEqual(["nub", "upgrade", "{installSpec}"]); + }); + + test("uses the setting when PI_SELF_UPDATE_COMMAND is unset or blank", () => { + process.env.PI_SELF_UPDATE_COMMAND = " "; + + expect(getConfiguredSelfUpdateArgv(["mise", "upgrade", "npm:{packageName}"])).toEqual([ + "mise", + "upgrade", + "npm:{packageName}", + ]); + expect(getConfiguredSelfUpdateArgv([])).toBeUndefined(); + expect(getConfiguredSelfUpdateArgv(undefined)).toBeUndefined(); + }); + + test("fills placeholders in every argument", () => { + const command = makeConfiguredSelfUpdateCommand( + ["wrapper", "install", "{installSpec}", "--name={packageName}", "--version", "{version}", "{unknown}"], + { packageName: "@earendil-works/pi-coding-agent", version: "1.2.3" }, + ); + + expect(command).toEqual({ + command: "wrapper", + args: [ + "install", + "@earendil-works/pi-coding-agent@1.2.3", + "--name=@earendil-works/pi-coding-agent", + "--version", + "1.2.3", + "{unknown}", + ], + display: + "wrapper install @earendil-works/pi-coding-agent@1.2.3 --name=@earendil-works/pi-coding-agent --version 1.2.3 {unknown}", + }); + }); +}); diff --git a/packages/coding-agent/test/custom-message.test.ts b/packages/coding-agent/test/custom-message.test.ts index cdc598010..3077a6288 100644 --- a/packages/coding-agent/test/custom-message.test.ts +++ b/packages/coding-agent/test/custom-message.test.ts @@ -23,7 +23,7 @@ describe("CustomMessageComponent", () => { }; const component = new CustomMessageComponent(message, renderer, undefined, 1); - expect(optionsSeen).toEqual([{ expanded: false, outputPad: 1 }]); + expect(optionsSeen).toMatchObject([{ expanded: false, outputPad: 1 }]); expect( component .render(40) @@ -33,7 +33,7 @@ describe("CustomMessageComponent", () => { component.setOutputPad(0); - expect(optionsSeen.at(-1)).toEqual({ expanded: false, outputPad: 0 }); + expect(optionsSeen.at(-1)).toMatchObject({ expanded: false, outputPad: 0 }); expect( component .render(40) diff --git a/packages/coding-agent/test/exec-runtime-limit.test.ts b/packages/coding-agent/test/exec-runtime-limit.test.ts new file mode 100644 index 000000000..879a45911 --- /dev/null +++ b/packages/coding-agent/test/exec-runtime-limit.test.ts @@ -0,0 +1,25 @@ +import * as buffer from "node:buffer"; +import { expect, it, vi } from "vitest"; +import { execCommand } from "../src/core/exec.ts"; + +vi.mock("node:buffer", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + // 32-bit Node's maximum is 16 bytes below 256 MiB. + constants: { ...actual.constants, MAX_STRING_LENGTH: 268_435_440 }, + }; +}); + +// Regression tests for https://github.com/earendil-works/pi/issues/11360 +it("executes with the default cap when the runtime string limit is below 256 MiB", async () => { + const result = await execCommand(process.execPath, ["-e", 'process.stdout.write("ok")'], process.cwd()); + + expect(result).toEqual({ stdout: "ok", stderr: "", code: 0, killed: false, truncated: false }); +}); + +it("still rejects an explicit cap above the runtime string limit", async () => { + await expect( + execCommand(process.execPath, ["-e", ""], process.cwd(), { maxBuffer: buffer.constants.MAX_STRING_LENGTH + 1 }), + ).rejects.toThrow(RangeError); +}); diff --git a/packages/coding-agent/test/exec-utf8-chunks.test.ts b/packages/coding-agent/test/exec-utf8-chunks.test.ts new file mode 100644 index 000000000..5dad0aae0 --- /dev/null +++ b/packages/coding-agent/test/exec-utf8-chunks.test.ts @@ -0,0 +1,53 @@ +import type * as ChildProcessModule from "node:child_process"; +import { ChildProcess, spawn } from "node:child_process"; +import { PassThrough } from "node:stream"; +import { setImmediate } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { execCommand } from "../src/core/exec.ts"; + +vi.mock("node:child_process", async (importOriginal) => ({ + ...(await importOriginal()), + spawn: vi.fn(), +})); + +// #11360: real streams control chunk boundaries without relying on OS pipe timing. +// Only process creation and termination are stubbed; decoding and process waiting are real. +it.each(["stdout", "stderr"] as const)( + "discards pending %s UTF-8 bytes when either stream exceeds the cap", + async (stream) => { + const child = new ChildProcess(); + const stdout = new PassThrough(); + const stderr = new PassThrough(); + child.stdout = stdout; + child.stderr = stderr; + vi.mocked(spawn).mockReturnValueOnce(child); + vi.spyOn(child, "kill").mockImplementation(() => { + queueMicrotask(() => { + child.emit("exit", null, "SIGTERM"); + child.emit("close", null, "SIGTERM"); + }); + return true; + }); + + const resultPromise = execCommand("test-command", [], process.cwd(), { maxBuffer: 3 }); + const output = stream === "stdout" ? stdout : stderr; + const otherOutput = stream === "stdout" ? stderr : stdout; + // The decoder first returns "a" and buffers the first byte of €. + output.write(Buffer.from([0x61, 0xe2])); + await setImmediate(); + // The other decoder also buffers the first byte of € before the shared cap is exceeded. + otherOutput.write(Buffer.from([0xe2])); + await setImmediate(); + output.write(Buffer.from([0x82, 0xac])); + + const result = await resultPromise; + expect(result).toMatchObject({ + stdout: stream === "stdout" ? "a" : "", + stderr: stream === "stderr" ? "a" : "", + truncated: true, + killed: true, + }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(3); + }, +); diff --git a/packages/coding-agent/test/exec.test.ts b/packages/coding-agent/test/exec.test.ts index 96232a8b3..df5a63817 100644 --- a/packages/coding-agent/test/exec.test.ts +++ b/packages/coding-agent/test/exec.test.ts @@ -1,3 +1,4 @@ +import { constants as bufferConstants } from "node:buffer"; import { describe, expect, it } from "vitest"; import { execCommand } from "../src/core/exec.ts"; @@ -9,6 +10,7 @@ describe("execCommand", () => { expect(result).toMatchObject({ code: 1, killed: false, + truncated: false, stdout: "", }); expect(result.stderr).toMatch(/ENOENT/); @@ -17,6 +19,110 @@ describe("execCommand", () => { it("keeps stderr empty when a command exits silently", async () => { const result = await execCommand(process.execPath, ["-e", "process.exit(1)"], process.cwd()); - expect(result).toEqual({ stdout: "", stderr: "", code: 1, killed: false }); + expect(result).toEqual({ stdout: "", stderr: "", code: 1, killed: false, truncated: false }); }); + + // Regression tests for https://github.com/earendil-works/pi/issues/11360 + it.each([ + -1, + 0.5, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + bufferConstants.MAX_STRING_LENGTH + 1, + ])("rejects invalid maxBuffer %s instead of attempting execution", async (maxBuffer) => { + // A missing command normally resolves to a failed result; invalid options must reject instead. + await expect( + execCommand("pi-command-that-does-not-exist-11360", [], process.cwd(), { maxBuffer }), + ).rejects.toThrow(RangeError); + }); + + it.each([0, bufferConstants.MAX_STRING_LENGTH])("accepts maxBuffer boundary %s", async (maxBuffer) => { + const result = await execCommand(process.execPath, ["-e", ""], process.cwd(), { maxBuffer }); + + expect(result).toEqual({ stdout: "", stderr: "", code: 0, killed: false, truncated: false }); + }); + + it("truncates all output when maxBuffer is zero", async () => { + const result = await execCommand(process.execPath, ["-e", 'process.stdout.write("x")'], process.cwd(), { + maxBuffer: 0, + }); + + expect(result).toMatchObject({ stdout: "", stderr: "", truncated: true, killed: true }); + expect(result.code).not.toBe(0); + }); + + it("caps combined output at maxBuffer, terminates the command, and reports truncation", async () => { + const script = + 'process.stderr.write("e".repeat(600)); process.stdout.write("o".repeat(600)); setInterval(() => process.stdout.write("x".repeat(1000)), 10);'; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer: 1000 }); + + expect(result).toMatchObject({ truncated: true, killed: true }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBe(1000); + }); + + // #11360: a byte cap must not turn a partial UTF-8 character into replacement output. + it.each([ + ["stdout", "€", 1, ""], + ["stderr", "€", 1, ""], + ["stdout", "a€🙂", 5, "a€"], + ["stderr", "a€🙂", 5, "a€"], + ] as const)( + "keeps complete UTF-8 characters from %s output %s capped at %i bytes", + async (stream, output, maxBuffer, expected) => { + const script = `process.${stream}.write(${JSON.stringify(output)})`; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer }); + + expect(result).toMatchObject({ [stream]: expected, truncated: true, killed: true }); + expect(result.code).not.toBe(0); + expect(Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr)).toBeLessThanOrEqual(maxBuffer); + }, + ); + + it("still flushes incomplete UTF-8 at normal EOF when output was not truncated", async () => { + const script = "process.stdout.write(Buffer.from([0xe2])); process.stderr.write(Buffer.from([0xf0]));"; + const result = await execCommand(process.execPath, ["-e", script], process.cwd()); + + expect(result).toEqual({ stdout: "�", stderr: "�", code: 0, killed: false, truncated: false }); + }); + + it("leaves output up to maxBuffer unchanged", async () => { + const result = await execCommand( + process.execPath, + ["-e", 'process.stdout.write("a".repeat(1000))'], + process.cwd(), + { + maxBuffer: 1000, + }, + ); + + expect(result).toEqual({ stdout: "a".repeat(1000), stderr: "", code: 0, killed: false, truncated: false }); + }); + + it.skipIf(process.platform === "win32")( + "returns when a shell's child keeps writing after the shell is terminated", + async () => { + // SIGTERM kills only `sh`; `yes` keeps the output pipes open and writing. + const result = await execCommand("sh", ["-c", "yes; echo done"], process.cwd(), { maxBuffer: 1000 }); + + expect(result).toMatchObject({ killed: true, truncated: true }); + expect(result.stdout).toBe("y\n".repeat(500)); + }, + 10_000, + ); + + it.skipIf(process.platform === "win32")( + "force-kills a command that ignores SIGTERM", + async () => { + // The handler is installed before the first write, so the kill triggered by maxBuffer + // cannot arrive before the child ignores SIGTERM. + const script = 'process.on("SIGTERM", () => {}); process.stdout.write("xx"); setInterval(() => {}, 1000);'; + const result = await execCommand(process.execPath, ["-e", script], process.cwd(), { maxBuffer: 1 }); + + expect(result).toMatchObject({ stdout: "x", killed: true, truncated: true }); + expect(result.code).not.toBe(0); + }, + 15_000, + ); }); diff --git a/packages/coding-agent/test/extension-keybinding-hints.test.ts b/packages/coding-agent/test/extension-keybinding-hints.test.ts new file mode 100644 index 000000000..7e0a3ebe9 --- /dev/null +++ b/packages/coding-agent/test/extension-keybinding-hints.test.ts @@ -0,0 +1,82 @@ +// Issue #4748: extensions that import keyText/keyHint can resolve their own package copy, which does not see the +// host's keybindings. The host passes its own keyText/keyHint to renderers instead. +import { setKeybindings, Text, type TUI } from "@earendil-works/pi-tui"; +import { Type } from "typebox"; +import { beforeAll, describe, expect, test } from "vitest"; +import type { KeybindingHints, ToolDefinition } from "../src/core/extensions/types.ts"; +import { KeybindingsManager } from "../src/core/keybindings.ts"; +import type { CustomMessage } from "../src/core/messages.ts"; +import type { CustomEntry } from "../src/core/session-manager.ts"; +import { CustomEntryComponent } from "../src/modes/interactive/components/custom-entry.ts"; +import { CustomMessageComponent } from "../src/modes/interactive/components/custom-message.ts"; +import { ToolExecutionComponent } from "../src/modes/interactive/components/tool-execution.ts"; +import { initTheme } from "../src/modes/interactive/theme/theme.ts"; +import { stripAnsi } from "../src/utils/ansi.ts"; + +function hintText(hints: KeybindingHints): string { + return `${hints.keyText("app.tools.expand")} | ${stripAnsi(hints.keyHint("app.tools.expand", "to expand"))}`; +} + +const EXPECTED = "ctrl+e | ctrl+e to expand"; + +describe("host keybinding hints for extensions", () => { + beforeAll(() => { + initTheme("dark"); + // A user override proves the hints read the host's configured keybindings, not the defaults. + setKeybindings(new KeybindingsManager({ "app.tools.expand": "ctrl+e" })); + }); + + test("tool renderers receive keyText and keyHint in the render context", () => { + const seen: string[] = []; + const toolDefinition: ToolDefinition = { + name: "custom_tool", + label: "custom_tool", + description: "custom tool", + parameters: Type.Any(), + execute: async () => ({ content: [{ type: "text", text: "ok" }], details: {} }), + renderCall: (_args, _theme, context) => { + seen.push(hintText(context)); + return new Text("call", 0, 0); + }, + }; + const tui = { requestRender: () => {} } as unknown as TUI; + new ToolExecutionComponent("custom_tool", "tool-1", {}, {}, toolDefinition, tui, process.cwd()).render(80); + + expect(seen.at(-1)).toBe(EXPECTED); + }); + + test("message renderers receive keyText and keyHint in the render options", () => { + const seen: string[] = []; + const message: CustomMessage = { + role: "custom", + customType: "test", + content: "custom", + display: true, + timestamp: Date.now(), + }; + new CustomMessageComponent(message, (_message, options) => { + seen.push(hintText(options)); + return new Text("custom", 0, 0); + }); + + expect(seen).toEqual([EXPECTED]); + }); + + test("entry renderers receive keyText and keyHint in the render options", () => { + const seen: string[] = []; + const entry: CustomEntry = { + type: "custom", + id: "entry-1", + parentId: null, + timestamp: new Date().toISOString(), + customType: "test", + data: undefined, + }; + new CustomEntryComponent(entry, (_entry, options) => { + seen.push(hintText(options)); + return new Text("custom", 0, 0); + }); + + expect(seen).toEqual([EXPECTED]); + }); +}); diff --git a/packages/coding-agent/test/external-editor.test.ts b/packages/coding-agent/test/external-editor.test.ts index 5cbbc93b3..e9a540821 100644 --- a/packages/coding-agent/test/external-editor.test.ts +++ b/packages/coding-agent/test/external-editor.test.ts @@ -52,7 +52,7 @@ describe("editInExternalEditor", () => { it("keeps the original content when the editor exits unsuccessfully", async () => { const { result, capture } = await runExternalEditor("--fail"); - expect(result).toEqual({ status: "failed" }); + expect(result).toEqual({ status: "failed", message: "External editor exited with code 1; prompt unchanged" }); expect(existsSync(dirname(capture.filePath))).toBe(false); }); it("returns empty content when the editor clears the prompt", async () => { @@ -60,4 +60,13 @@ describe("editInExternalEditor", () => { expect(result).toEqual({ status: "complete", content: "" }); }); + + // #11592: launch failures must be reported, not silently ignored. + it.skipIf(process.platform === "win32")("reports an editor that cannot be launched", async () => { + const command = "pi-external-editor-does-not-exist"; + expect(await editInExternalEditor({ command, content: "original" })).toEqual({ + status: "failed", + message: `Failed to launch external editor: ${command}`, + }); + }); }); diff --git a/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts b/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts index 7c8741c7e..7903a4078 100644 --- a/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts +++ b/packages/coding-agent/test/git-merge-and-resolve-extension.test.ts @@ -7,8 +7,8 @@ import type { ExecResult, ExtensionAPI, ExtensionContext } from "../src/core/ext type AgentEndHandler = (event: { type: "agent_end" }, ctx: ExtensionContext) => Promise; -const ok: ExecResult = { stdout: "", stderr: "", code: 0, killed: false }; -const fail: ExecResult = { stdout: "", stderr: "error", code: 1, killed: false }; +const ok: ExecResult = { stdout: "", stderr: "", code: 0, killed: false, truncated: false }; +const fail: ExecResult = { stdout: "", stderr: "error", code: 1, killed: false, truncated: false }; /** Standard exec results for a clean repo tracking origin/main, not in a merge. */ function withUpstream(results: Map): Map { diff --git a/packages/coding-agent/test/git-update.test.ts b/packages/coding-agent/test/git-update.test.ts index 80308277c..849383aa2 100644 --- a/packages/coding-agent/test/git-update.test.ts +++ b/packages/coding-agent/test/git-update.test.ts @@ -150,8 +150,13 @@ describe("DefaultPackageManager git update", () => { } }; + const updated: string[] = []; + packageManager.setProgressCallback((event) => { + if (event.type === "result" && event.action === "update" && event.message) updated.push(event.message); + }); await packageManager.update(); + expect(updated).toEqual([]); expect(executedCommands).toContain( "git fetch --prune --no-tags origin +refs/heads/main:refs/remotes/origin/main", ); @@ -165,19 +170,78 @@ describe("DefaultPackageManager git update", () => { it("should update to latest commit when remote has new commits", async () => { setupRemoteAndInstall(); expect(getFileContent(installedDir, "extension.ts")).toBe("// v1"); + const oldCommit = getCurrentCommit(installedDir); // Add a new commit to remote const newCommit = createCommit(remoteDir, "extension.ts", "// v2", "Second commit"); // Update via package manager (no args = uses settings) + const updated: string[] = []; + packageManager.setProgressCallback((event) => { + if (event.type === "result" && event.action === "update" && event.message) updated.push(event.message); + }); await packageManager.update(); // Verify update succeeded + expect(updated).toEqual([`${gitSource} ${oldCommit.slice(0, 7)} -> ${newCommit.slice(0, 7)}`]); expect(getCurrentCommit(installedDir)).toBe(newCommit); expect(getFileContent(installedDir, "extension.ts")).toBe("// v2"); }); }); + describe("git clean failures", () => { + // https://github.com/earendil-works/pi/issues/11597 + it("should install dependencies and retry the clean later when git clean fails", async () => { + setupRemoteAndInstall(); + const newCommit = createCommit( + remoteDir, + "package.json", + JSON.stringify({ name: "test-extension", version: "1.0.0" }), + "Add package.json", + ); + + const executedCommands: string[] = []; + let failClean = true; + const managerWithInternals = packageManager as unknown as { + runCommand: (command: string, args: string[], options?: { cwd?: string }) => Promise; + }; + managerWithInternals.runCommand = async (command, args, options) => { + const commandLine = `${command} ${args.join(" ")}`; + executedCommands.push(commandLine); + if (command === "npm") return; + if (commandLine === "git clean -fdx" && failClean) { + throw new Error("git clean -fdx failed with code 1"); + } + const result = spawnSync(command, args, { cwd: options?.cwd, encoding: "utf-8" }); + if (result.status !== 0) { + throw new Error(`Command failed: ${commandLine}\n${result.stderr}`); + } + }; + const progressMessages: string[] = []; + packageManager.setProgressCallback((event) => { + if (event.type === "progress" && event.message) progressMessages.push(event.message); + }); + const markerPath = join(agentDir, "git", "github.com", "test", ".extension.pi-update-incomplete"); + + await packageManager.update(); + + expect(getCurrentCommit(installedDir)).toBe(newCommit); + expect(executedCommands).toContain("git clean -fdx"); + expect(executedCommands.some((command) => command.startsWith("npm install"))).toBe(true); + expect(progressMessages.some((message) => message.includes("Could not remove all untracked files"))).toBe( + true, + ); + expect(existsSync(markerPath)).toBe(true); + + failClean = false; + executedCommands.length = 0; + await packageManager.update(); + + expect(executedCommands).toContain("git clean -fdx"); + expect(existsSync(markerPath)).toBe(false); + }); + }); + describe("force-push scenarios", () => { it("should handle complete history rewrite", async () => { setupRemoteAndInstall(); diff --git a/packages/coding-agent/test/image-processing.test.ts b/packages/coding-agent/test/image-processing.test.ts index 1bf3ea53e..0d112a3a7 100644 --- a/packages/coding-agent/test/image-processing.test.ts +++ b/packages/coding-agent/test/image-processing.test.ts @@ -5,6 +5,8 @@ import { describe, expect, it } from "vitest"; import { convertToPng, loadPngTranscoder } from "../src/utils/image-convert.ts"; import { formatDimensionNote, resizeImage } from "../src/utils/image-resize.ts"; +import { resizeImageInProcess } from "../src/utils/image-resize-core.ts"; +import { loadPhoton } from "../src/utils/photon.ts"; // Small 2x2 red PNG image (base64) - generated with ImageMagick const TINY_PNG = @@ -25,6 +27,31 @@ const MEDIUM_PNG_100x100 = const LARGE_PNG_200x200 = "iVBORw0KGgoAAAANSUhEUgAAAMgAAADIEAIAAAByquWKAAAAIGNIUk0AAHomAACAhAAA+gAAAIDoAAB1MAAA6mAAADqYAAAXcJy6UTwAAAAGYktHRP///////wlY99wAAAAHdElNRQfqAQ4AMzqU03BFAAAEEElEQVR42u3dvWqUURSF4XXMRAyIhSAWQvwnWtjaWusFx9bW1kKD/wELESxEiBjDjJdwysOC57mCXW32+zGcGbtdACpsdi+T8Xz1GABzm91xMl6sHgNgblzcT/Y+rh4DYG6cv0/2H64eA2BOEgI1fHQHaoyzbXIwVo8BMCcJgRqSEKgxft9Lrn5aPQbA3Ph1klw7Wj0GwJwkBGpYWECN8fMiub63egyAORcWUMPCAmqMH3eTG59XjwEwN76/S24+Wj0GwJwkBGpYWECN8e1fcmuzegyAORcWUMPCAmqM09vJ4dfVYwDMjS9vkzuPV48BMCcJgRqeSAZqjA/nyYP91WMAzElCoIYkBGqMk8Pk6HT1GABzm+3L5JIkBApIQqCGj+5AjfHmb/Lk8uoxAOYkIVBDEgI1LCygxmZ7nFyShEABFxZQw8ICaozXf5KnV1aPATDnwgJqWFhADQsLqOFnDUANFxZQw8ICaoxXZ8mzg9VjAMy5sIAaFhZQw3tYQA1/QgHUkIRADUkI1JCEQA1JCNSQhEANFxZQwzcsoIYkBGpIQqCGJARqSEKghiQEalhYQA1/QgHUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEaFhZQw4ujQA0XFlDDwgJqSEKghgsLqGFhATUsLKCGb1hADRcWUMPCAmpIQqCGCwuoYWEBNbyHBdTw4ihQQxICNSQhUEMSAjUkIVBDEgI1XFhADd+wgBqSEKghCYEakhCoIQmBGpIQqGFhATW8OArUcGEBNSwsoIYkBGq4sIAaFhZQw8ICaviGBdRwYQE1LCyghiQEariwgBoWFlDDe1hADS+OAjUkIVBDEgI1JCFQQxICNSQhUMOFBdTwDQuoIQmBGpIQqCEJgRqSEKghCYEa49lB8ups9RgAc14cBWpIQqCGhQXUkIRADRcWUMPCAmqMp1eS139WjwEw5xsWUEMSAjUsLKCGJARquLCAGhYWUGM8uZy8+bt6DIA5L44CNSQhUMMTyUCNcXSanByuHgNgThICNSQhUGM82E8+nK8eA2DOzxqAGpIQqDHuPE6+vF09BsDcOPyanN5ePQbAnCQEavgdFlBj3Nok3/6tHgNgzoujQA1JCNQYNx8l39+tHgNgbtz4nPy4u3oMgDlJCNSwsIAa4/pe8vNi9RgAc37WANSQhECNce0o+XWyegyAuXH1U/L73uoxAOYkIVDDwgJqjIORnG1XjwEw508ogBqSEKgx9h8m5+9XjwEwN/Y+Jhf3V48BMCcJgRpjPE+2x6vHAJgbSbLbrR4DYO4/GqiSgXN+ksgAAAAldEVYdGRhdGU6Y3JlYXRlADIwMjYtMDEtMTRUMDA6NTE6NTcrMDA6MDDpysx4AAAAJXRFWHRkYXRlOm1vZGlmeQAyMDI2LTAxLTE0VDAwOjUxOjU3KzAwOjAwmJd0xAAAACh0RVh0ZGF0ZTp0aW1lc3RhbXAAMjAyNi0wMS0xNFQwMDo1MTo1NyswMDowMM+CVRsAAAAASUVORK5CYII="; +/** Photo-like JPEG: smooth gradients with light deterministic noise, so PNG compresses it poorly. */ +async function photoLikeJpeg(width: number, height: number): Promise { + const photon = await loadPhoton(); + if (!photon) throw new Error("Photon not available"); + const pixels = new Uint8Array(width * height * 4); + let seed = 1; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + seed = (seed * 1103515245 + 12345) & 0x7fffffff; + const noise = seed % 6; + const i = (y * width + x) * 4; + pixels[i] = ((x / width) * 200 + noise) | 0; + pixels[i + 1] = ((y / height) * 200 + noise) | 0; + pixels[i + 2] = (128 + Math.sin(x / 30) * 60 + noise) | 0; + pixels[i + 3] = 255; + } + } + const image = new photon.PhotonImage(pixels, width, height); + try { + return image.get_bytes_jpeg(85); + } finally { + image.free(); + } +} + function imageBytes(base64Data: string): Uint8Array { return Buffer.from(base64Data, "base64"); } @@ -166,6 +193,18 @@ describe("resizeImage", () => { expect(result).toBeNull(); }); + // Regression test for #11551: PNG was returned whenever it fit, even when JPEG was much smaller. + it("should pick JPEG over a larger PNG when resizing a photo", async () => { + const input = await photoLikeJpeg(1200, 800); + const result = await resizeImageInProcess(input, "image/jpeg", { maxWidth: 1000, maxHeight: 1000 }); + + expect(result).not.toBeNull(); + expect(result!.wasResized).toBe(true); + expect(result!.mimeType).toBe("image/jpeg"); + expect(result!.width).toBe(1000); + expect(result!.data.length).toBeLessThan(Math.ceil(input.byteLength / 3) * 4 * 2); + }); + it("should handle JPEG input", async () => { const result = await resizeImage(imageBytes(TINY_JPEG), "image/jpeg", { maxWidth: 100, diff --git a/packages/coding-agent/test/input-transform-streaming-example.test.ts b/packages/coding-agent/test/input-transform-streaming-example.test.ts index faf95b6ba..0eb1ccf60 100644 --- a/packages/coding-agent/test/input-transform-streaming-example.test.ts +++ b/packages/coding-agent/test/input-transform-streaming-example.test.ts @@ -35,9 +35,9 @@ function setup(execResult: ExecResult) { describe("input-transform-streaming example", () => { const diffOutput = " src/index.ts | 5 ++---\n 1 file changed, 2 insertions(+), 3 deletions(-)"; - const gitSuccess: ExecResult = { stdout: diffOutput, stderr: "", code: 0, killed: false }; - const gitEmpty: ExecResult = { stdout: "", stderr: "", code: 0, killed: false }; - const gitFail: ExecResult = { stdout: "", stderr: "not a git repo", code: 128, killed: false }; + const gitSuccess: ExecResult = { stdout: diffOutput, stderr: "", code: 0, killed: false, truncated: false }; + const gitEmpty: ExecResult = { stdout: "", stderr: "", code: 0, killed: false, truncated: false }; + const gitFail: ExecResult = { stdout: "", stderr: "not a git repo", code: 128, killed: false, truncated: false }; it("skips exec during steering", async () => { const { emit, exec } = setup(gitSuccess); diff --git a/packages/coding-agent/test/interactive-mode-status.test.ts b/packages/coding-agent/test/interactive-mode-status.test.ts index 72de1fa96..3e26d845c 100644 --- a/packages/coding-agent/test/interactive-mode-status.test.ts +++ b/packages/coding-agent/test/interactive-mode-status.test.ts @@ -1366,7 +1366,7 @@ describe("InteractiveMode.showLoadedResources", () => { }); const output = renderAll(fakeThis.loadedResourcesContainer); - expect(output).toContain("[Skill conflicts]"); + expect(output).toContain("[Skill warnings]"); expect(output).not.toContain("[Skills]"); }); }); diff --git a/packages/coding-agent/test/mcp-command.test.ts b/packages/coding-agent/test/mcp-command.test.ts index 7bde1e11b..2e9a0a1fa 100644 --- a/packages/coding-agent/test/mcp-command.test.ts +++ b/packages/coding-agent/test/mcp-command.test.ts @@ -2,6 +2,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { ProjectTrustStore } from "../src/core/trust-manager.ts"; import { runMcpCommand } from "../src/extensions/mcp/cli.ts"; const FIXTURE = resolve(import.meta.dirname, "../../mcp/test/fixtures/stdio-server.mjs"); @@ -74,6 +75,30 @@ describe("pi mcp", () => { expect((await run(["frobnicate"], servers)).exitCode).toBe(1); }); + it("suggests a configured server for a mistyped name", async () => { + expect((await run(["login", "fixtures"], servers)).output).toBe( + 'No MCP server named "fixtures". Did you mean "fixture"? Configured: fixture, broken, parked.', + ); + }); + + it("reports options as typed", async () => { + expect((await run(["list", "-l"], servers)).output).toContain("Unknown option -l."); + }); + + it("restricts login and logout to project servers with --local", async () => { + const { agentDir } = await run(["add", "-l", "local", "--", "node", "server.js"], servers); + new ProjectTrustStore(agentDir).set(agentDir, true); + + expect(await run(["logout", "-l", "local"], undefined, agentDir)).toMatchObject({ + exitCode: 1, + output: 'MCP server "local" does not use OAuth. Only HTTP servers without an Authorization header do.', + }); + expect(await run(["login", "-l", "fixture"], undefined, agentDir)).toMatchObject({ + exitCode: 1, + output: `MCP server "fixture" is defined in ${join(agentDir, "mcp.json")}, not in the project; omit --local.`, + }); + }); + it("adds stdio servers and passes options after the command through", async () => { const added = await run( ["add", "--env", "A=1", "--env", "B=x=y", "files", "--", "npx", "-y", "server", "--root", "."], @@ -146,6 +171,15 @@ describe("pi mcp", () => { expect(readConfig(join(agentDir, "mcp.json")).mcpServers).toMatchObject({ sentry: { url: "https://mcp.sentry.dev/mcp", oauth: { clientId: "pi", clientName: "Claude Code" } }, }); + + await run( + ["add", "linear", "--url", "https://mcp.linear.app/mcp", "--oauth-client-registration", "dcr"], + undefined, + agentDir, + ); + expect(readConfig(join(agentDir, "mcp.json")).mcpServers).toMatchObject({ + linear: { url: "https://mcp.linear.app/mcp", oauth: { clientRegistration: "dcr" } }, + }); }); it("rejects invalid add invocations without writing", async () => { @@ -158,6 +192,7 @@ describe("pi mcp", () => { ["add", "x", "--header", "A=1", "--", "cmd"], ["add", "x", "--env", "NOVALUE", "--", "cmd"], ["add", "x", "--exposure", "loud", "--", "cmd"], + ["add", "x", "--url", "https://example.com", "--oauth-client-registration", "maybe"], ]; for (const args of cases) { const result = await run(args, undefined); diff --git a/packages/coding-agent/test/mcp-conformance/README.md b/packages/coding-agent/test/mcp-conformance/README.md index 8aaa36690..dbdbc5bf1 100644 --- a/packages/coding-agent/test/mcp-conformance/README.md +++ b/packages/coding-agent/test/mcp-conformance/README.md @@ -39,7 +39,7 @@ node packages/coding-agent/test/mcp-conformance/run.ts --update-baseline ## Known failures - `elicitation-sep1034-client-defaults`: pi does not support elicitation. -- `auth/basic-cimd`: pi registers clients dynamically instead of using a Client ID Metadata Document. +- `auth/basic-cimd`: pi uses its Client ID Metadata Document on pi.dev, not the suite's `https://conformance-test.local/client-metadata.json`. - `auth/scope-retry-limit`: `pi-client` fails by design; the server never accepts the granted scope. 2026-07-28 is not covered: it is a stateless protocol pi does not implement. diff --git a/packages/coding-agent/test/mcp-conformance/baseline.json b/packages/coding-agent/test/mcp-conformance/baseline.json index 0401d64b0..8e7f205ae 100644 --- a/packages/coding-agent/test/mcp-conformance/baseline.json +++ b/packages/coding-agent/test/mcp-conformance/baseline.json @@ -214,7 +214,6 @@ "passing": [ "authorization-request", "authorization-server-metadata", - "client-registration", "pi-client", "pkce-code-challenge-sent", "pkce-code-verifier-sent", diff --git a/packages/coding-agent/test/mcp-extension.test.ts b/packages/coding-agent/test/mcp-extension.test.ts index 0948434fd..7bc18ca43 100644 --- a/packages/coding-agent/test/mcp-extension.test.ts +++ b/packages/coding-agent/test/mcp-extension.test.ts @@ -208,7 +208,8 @@ describe("MCP config", () => { url: "https://a.example/mcp", oauth: { clientRegistration: "cimd", callbackUrl: "http://localhost/callback" }, }, - badRegistration: { url: "https://a.example/mcp", oauth: { clientRegistration: "auto" } }, + auto: { url: "https://a.example/mcp", oauth: { clientRegistration: "auto", clientName: "x" } }, + badRegistration: { url: "https://a.example/mcp", oauth: { clientRegistration: "oidc" } }, cimdClient: { url: "https://a.example/mcp", oauth: { clientRegistration: "cimd", clientId: "x" } }, cimdPath: { url: "https://a.example/mcp", @@ -219,14 +220,14 @@ describe("MCP config", () => { {}, ); const { servers, errors } = loadMcpConfig({ ...paths, projectTrusted: false }); - expect(servers.map((server) => server.name)).toEqual(["ok", "ipv6", "same", "named", "metadata", "cimd"]); + expect(servers.map((server) => server.name)).toEqual(["ok", "ipv6", "same", "named", "metadata", "cimd", "auto"]); expect(errors).toEqual([ expect.stringContaining('server "remote": oauth.callbackUrl must be an http URI on localhost'), expect.stringContaining('server "both": oauth.callbackUrl and oauth.callbackPort name different ports'), expect.stringContaining('server "scope": oauth.scope must be a string'), expect.stringContaining('server "unnamed": oauth.clientName must be a non-empty string'), expect.stringContaining('server "plainMetadata": oauth.authServerMetadataUrl must be an https URL'), - expect.stringContaining('server "badRegistration": oauth.clientRegistration must be "dcr" or "cimd"'), + expect.stringContaining('server "badRegistration": oauth.clientRegistration must be "auto", "dcr", or "cimd"'), expect.stringContaining('server "cimdClient": oauth.clientRegistration "cimd" cannot be combined'), expect.stringContaining('server "cimdPath": oauth.clientRegistration "cimd" requires oauth.callbackUrl'), ]); diff --git a/packages/coding-agent/test/mcp-oauth-refresh.test.ts b/packages/coding-agent/test/mcp-oauth-refresh.test.ts index a901ec35a..aed142a87 100644 --- a/packages/coding-agent/test/mcp-oauth-refresh.test.ts +++ b/packages/coding-agent/test/mcp-oauth-refresh.test.ts @@ -164,6 +164,7 @@ describe("MCP OAuth sign-in", () => { describe("MCP OAuth client ID metadata documents", () => { const cleanups: (() => Promise)[] = []; const cimd: McpOAuthSettings = { clientRegistration: "cimd" }; + const dcr: McpOAuthSettings = { clientRegistration: "dcr" }; afterEach(async () => { while (cleanups.length > 0) await cleanups.pop()?.(); @@ -183,16 +184,17 @@ describe("MCP OAuth client ID metadata documents", () => { return { server, store, signIn }; } - it("registers dynamically by default, even when the server supports documents", async () => { + it("registers dynamically with dcr, even when the server supports documents", async () => { const { server, signIn } = await startServer({ cimd: true, issParameter: true }); - await signIn({}); + await signIn(dcr); expect(server.registrations).toHaveLength(1); expect(server.authorizations[0].get("client_id")).toBe("client-1"); }); - it("uses pi's document when authorization responses name their issuer", async () => { + // #11426 + it("uses pi's document by default when authorization responses name their issuer", async () => { const { server, store, signIn } = await startServer({ cimd: true, issParameter: true }); - await signIn(cimd); + await signIn({}); const [authorization] = server.authorizations; expect(authorization.get("client_id")).toBe("https://pi.dev/oauth/client.json"); const redirect = new URL(authorization.get("redirect_uri") ?? ""); @@ -203,12 +205,39 @@ describe("MCP OAuth client ID metadata documents", () => { expect(server.tokenRequests[0].get("redirect_uri")).toBe(redirect.href); // The document is not stored, so signing in again refreshes the tokens instead of discarding them. expect((await store.load())?.clientInformation).toBeUndefined(); - await signIn(cimd); + await signIn({}); expect(server.authorizations).toHaveLength(1); expect(server.tokenRequests[1].get("grant_type")).toBe("refresh_token"); expect(server.tokenRequests[1].get("client_id")).toBe("https://pi.dev/oauth/client.json"); }); + // #11426 + it("registers dynamically by default when the server does not support documents", async () => { + const { server, signIn } = await startServer({ issParameter: true }); + await signIn({}); + expect(server.registrations).toHaveLength(1); + expect(server.authorizations[0].get("client_id")).toBe("client-1"); + }); + + // #11426 + it("registers dynamically by default with a client name", async () => { + const { server, signIn } = await startServer({ cimd: true, issParameter: true }); + await signIn({ clientName: "Claude Code" }); + expect(server.registrations).toMatchObject([{ client_name: "Claude Code" }]); + }); + + // #11426 + it("keeps a registered client by default", async () => { + const { server, store, signIn } = await startServer({ cimd: true, issParameter: true }); + await signIn(dcr); + await signIn({}); + // The registered client's tokens are refreshed instead of signing in with the document. + expect(server.authorizations).toHaveLength(1); + expect(server.tokenRequests[1].get("grant_type")).toBe("refresh_token"); + expect(server.tokenRequests[1].get("client_id")).toBe("client-1"); + expect((await store.load())?.clientInformation?.client_id).toBe("client-1"); + }); + it("uses a document and callback path specific to the MCP server without the iss parameter", async () => { const { server, signIn } = await startServer({ cimd: true }); await signIn(cimd); @@ -224,7 +253,7 @@ describe("MCP OAuth client ID metadata documents", () => { it("replaces a registered client when switching to the document", async () => { const { server, store, signIn } = await startServer({ cimd: true, issParameter: true }); - await signIn({}); + await signIn(dcr); expect((await store.load())?.clientInformation?.client_id).toBe("client-1"); await signIn(cimd); // The registered client's tokens are not refreshed with another client. diff --git a/packages/coding-agent/test/model-catalog-protocol.test.ts b/packages/coding-agent/test/model-catalog-protocol.test.ts index 9222af565..197cab282 100644 --- a/packages/coding-agent/test/model-catalog-protocol.test.ts +++ b/packages/coding-agent/test/model-catalog-protocol.test.ts @@ -61,8 +61,6 @@ function startCatalogServer(requests: string[]): Promise { const catalogRequest = parseModelCatalogRequest(url, request.headers["user-agent"]); if (!provider) { response.writeHead(404).end(); - } else if (catalogRequest.kind === "redirect") { - response.writeHead(307, { location: catalogRequest.location, "cache-control": "no-store" }).end(); } else if (catalogRequest.kind === "invalid") { response.writeHead(400).end(catalogRequest.error); } else { @@ -120,7 +118,7 @@ describe("model catalog protocol with the current client", () => { expect([...refresh.errors]).toEqual([]); const catalogUrl = "/api/models/providers/openrouter?types=chat%2Cimage%2Cclassifier"; - expect(requests).toEqual([catalogUrl, `${catalogUrl}&pi-version=${VERSION}`]); + expect(requests).toEqual([catalogUrl]); const expectedModel = selectModelCatalog(index, VERSION)?.revision === legacyRevision ? legacyModel : mixedApiModel; diff --git a/packages/coding-agent/test/package-command-paths.test.ts b/packages/coding-agent/test/package-command-paths.test.ts index 68949f9a3..711cf3f96 100644 --- a/packages/coding-agent/test/package-command-paths.test.ts +++ b/packages/coding-agent/test/package-command-paths.test.ts @@ -369,6 +369,54 @@ if (process.platform !== "win32") fs.chmodSync(piPath, 0o755); } }); + /** Install `npm:fake-package` at `installedVersion` with an npm whose registry reports `registryVersion`. */ + function prepareStaleNpmPackage(installedVersion: string, registryVersion: string): void { + const installedPath = join(agentDir, "npm", "node_modules", "fake-package"); + mkdirSync(installedPath, { recursive: true }); + writeFileSync( + join(installedPath, "package.json"), + JSON.stringify({ name: "fake-package", version: installedVersion }), + ); + // `npm view` reports the newer version, but `npm install` succeeds without changing anything. + const fakeNpmPath = join(tempDir, "fake-stale-npm.cjs"); + writeFileSync( + fakeNpmPath, + `if (process.argv.includes("view")) console.log(${JSON.stringify(JSON.stringify(registryVersion))});`, + ); + writeFileSync( + join(agentDir, "settings.json"), + JSON.stringify({ packages: ["npm:fake-package"], npmCommand: [originalExecPath, fakeNpmPath] }), + ); + } + + it("does not report packages as up to date when npm left an older version installed", async () => { + prepareStaleNpmPackage("1.2.0", "1.3.0"); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const stdoutSpy = vi.spyOn(process.stdout, "write").mockImplementation(() => true); + + await expect(main(["update", "--extensions"])).resolves.toBeUndefined(); + + const progress = stdoutSpy.mock.calls.map(([chunk]) => String(chunk)).join(""); + const stdout = logSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(progress).toContain("npm:fake-package is still 1.2.0 (registry reports 1.3.0)"); + expect(stdout).toContain("No packages were updated, see the warnings above"); + expect(stdout).not.toContain("up to date"); + }); + + it("does not report packages as up to date in offline mode", async () => { + prepareStaleNpmPackage("1.2.0", "1.3.0"); + // main() writes both variables in offline mode; stubbing them lets afterEach restore them. + vi.stubEnv("PI_OFFLINE", "1"); + vi.stubEnv("PI_SKIP_VERSION_CHECK", "1"); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await expect(main(["update", "--extensions"])).resolves.toBeUndefined(); + + const stdout = logSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stdout).toContain("Skipped package updates (offline mode)"); + expect(stdout).not.toContain("up to date"); + }); + it("uses saved project trust during update", async () => { mkdirSync(join(projectDir, ".pi"), { recursive: true }); const fakeNpmPath = join(tempDir, "fake-trusted-project-npm.cjs"); @@ -774,6 +822,140 @@ if (process.platform !== "win32") fs.chmodSync(piPath, 0o755); expect(process.exitCode).toBe(1); }); + function writeNodeShim(binDir: string, name: string, scriptPath: string): void { + const shimPath = join(binDir, process.platform === "win32" ? `${name}.cmd` : name); + writeFileSync( + shimPath, + process.platform === "win32" + ? `@echo off\r\n"${originalExecPath}" "${scriptPath}" %*\r\n` + : `#!/bin/sh\nexec "${originalExecPath}" "${scriptPath}" "$@"\n`, + ); + chmodSync(shimPath, 0o755); + } + + /** Put a `pi-wrapper` update command and a `pi` that prints `piVersion` first on PATH. */ + function prepareSelfUpdateCommand(piVersion: string, exitCode = 0): { scriptPath: string; recordPath: string } { + const binDir = join(tempDir, "self-update-bin"); + const scriptPath = join(tempDir, "self-update.cjs"); + const piScriptPath = join(tempDir, "fake-pi.cjs"); + const recordPath = join(tempDir, "self-update-record.json"); + mkdirSync(binDir, { recursive: true }); + writeFileSync( + scriptPath, + `require("node:fs").writeFileSync(${JSON.stringify(recordPath)}, JSON.stringify(process.argv.slice(2))); +process.exit(${exitCode}); +`, + ); + writeFileSync(piScriptPath, `console.log(${JSON.stringify(piVersion)});\n`); + writeNodeShim(binDir, "pi-wrapper", scriptPath); + writeNodeShim(binDir, "pi", piScriptPath); + process.env.PATH = `${binDir}${delimiter}${originalPath ?? ""}`; + return { scriptPath, recordPath }; + } + + it("runs PI_SELF_UPDATE_COMMAND instead of the managed update and the selfUpdateCommand setting", async () => { + const targetVersion = getNewerPatchVersion(); + const { npmRecordPath } = prepareManagedInstall(targetVersion); + const { scriptPath, recordPath } = prepareSelfUpdateCommand(targetVersion); + const settingRecordPath = join(tempDir, "setting-record.json"); + writeFileSync( + join(agentDir, "settings.json"), + JSON.stringify({ selfUpdateCommand: [originalExecPath, scriptPath, settingRecordPath] }), + ); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", "pi-wrapper upgrade {installSpec} {version}"); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ packageName: "@new-scope/pi", version: targetVersion, note: "Read me" })), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + + expect(JSON.parse(readFileSync(recordPath, "utf8")) as string[]).toEqual([ + "upgrade", + `@new-scope/pi@${targetVersion}`, + targetVersion, + ]); + expect(existsSync(npmRecordPath)).toBe(false); + const stdout = logSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stdout).toContain("Read me"); + // Like mise, the command leaves this installation untouched; the pi on PATH reports the new version. + expect(stdout).toContain(`Updated pi from ${VERSION} to ${targetVersion}`); + expect(errorSpy).not.toHaveBeenCalled(); + expect(process.exitCode).toBeUndefined(); + }); + + it("runs the global selfUpdateCommand setting only when a newer release exists", async () => { + const targetVersion = getNewerPatchVersion(); + const { scriptPath, recordPath } = prepareSelfUpdateCommand(targetVersion); + const projectRecordPath = join(tempDir, "project-record.json"); + writeFileSync( + join(agentDir, "settings.json"), + JSON.stringify({ selfUpdateCommand: [originalExecPath, scriptPath, "{packageName}"] }), + ); + mkdirSync(join(projectDir, ".pi"), { recursive: true }); + writeFileSync( + join(projectDir, ".pi", "settings.json"), + JSON.stringify({ selfUpdateCommand: [originalExecPath, scriptPath, projectRecordPath] }), + ); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", ""); + const fetchMock = vi.fn(async () => Response.json({ version: VERSION })); + vi.stubGlobal("fetch", fetchMock); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + expect(existsSync(recordPath)).toBe(false); + expect(logSpy.mock.calls.map(([message]) => String(message)).join("\n")).toContain( + `pi is already up to date (v${VERSION})`, + ); + + fetchMock.mockImplementation(async () => Response.json({ version: targetVersion })); + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + expect(JSON.parse(readFileSync(recordPath, "utf8")) as string[]).toEqual([PACKAGE_NAME]); + expect(process.exitCode).toBeUndefined(); + }); + + it("warns when the pi on PATH does not report the new version", async () => { + const targetVersion = getNewerPatchVersion(); + prepareSelfUpdateCommand(VERSION); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", "pi-wrapper upgrade"); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ version: targetVersion })), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + + const stdout = logSpy.mock.calls.map(([message]) => String(message)).join("\n"); + expect(stdout).toContain( + `Ran pi-wrapper upgrade, but \`pi --version\` reports ${VERSION}, not ${targetVersion}.`, + ); + expect(stdout).not.toContain("Updated pi from"); + expect(process.exitCode).toBeUndefined(); + }); + + it("fails when the configured self-update command fails", async () => { + prepareSelfUpdateCommand(VERSION, 23); + vi.stubEnv("PI_SELF_UPDATE_COMMAND", "pi-wrapper upgrade"); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ version: getNewerPatchVersion() })), + ); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect(runPackageCommandDirectly(["update", "--self"])).resolves.toBeUndefined(); + + expect(errorSpy.mock.calls.map(([message]) => String(message)).join("\n")).toContain( + "pi-wrapper upgrade exited with code 23", + ); + expect(logSpy.mock.calls.map(([message]) => String(message)).join("\n")).not.toContain("Ran pi-wrapper"); + expect(process.exitCode).toBe(1); + }); + it("keeps the managed release active when its update fails", async () => { const targetVersion = getNewerPatchVersion(); const { managedRoot } = prepareManagedInstall(targetVersion, 23); diff --git a/packages/coding-agent/test/package-manager.test.ts b/packages/coding-agent/test/package-manager.test.ts index 6aaeb7773..7d01a4ece 100644 --- a/packages/coding-agent/test/package-manager.test.ts +++ b/packages/coding-agent/test/package-manager.test.ts @@ -1211,7 +1211,8 @@ Content`, expect(runCommandSpy).not.toHaveBeenCalledWith("git", ["clean", "-fdx"], { cwd: targetDir }); }); - it("should repair deleted git package dependencies when cleaning fails", async () => { + // https://github.com/earendil-works/pi/issues/11597 + it("should install git package dependencies when cleaning fails", async () => { const source = "git:github.com/user/repo"; const targetDir = join(agentDir, "git", "github.com", "user", "repo"); const fetchArgs = ["fetch", "--prune", "--no-tags", "origin", "+refs/heads/main:refs/remotes/origin/main"]; @@ -1237,7 +1238,7 @@ Content`, if (args[0] === "clean") throw new Error("simulated clean failure"); }); - await expect(packageManager.update(source)).rejects.toThrow("simulated clean failure"); + await packageManager.update(source); expect(runCommandSpy).toHaveBeenCalledWith( "npm", @@ -2611,10 +2612,21 @@ export default function(api) { api.registerTool({ name: "test", description: "te const runCommandCaptureSpy = vi .spyOn(packageManager as any, "runCommandCapture") .mockResolvedValue('["1.0.0","1.2.0"]'); - const runCommandSpy = vi.spyOn(packageManager as any, "runCommand").mockResolvedValue(undefined); + const runCommandSpy = vi.spyOn(packageManager as any, "runCommand").mockImplementation(async () => { + writeFileSync(join(installedPath, "package.json"), JSON.stringify({ name: "example", version: "1.2.0" })); + }); + const events: ProgressEvent[] = []; + packageManager.setProgressCallback((event) => events.push(event)); await packageManager.update("npm:example"); + // The result is a separate event, so each start still has exactly one complete. + expect(events.map((event) => [event.type, event.source, event.message])).toEqual([ + ["start", "npm:example@^1.0.0", "Updating npm:example@^1.0.0..."], + ["complete", "npm:example@^1.0.0", undefined], + ["result", "npm:example@^1.0.0", "npm:example@^1.0.0 (project) 1.0.0 -> 1.2.0"], + ]); + expect(runCommandCaptureSpy).toHaveBeenCalledWith( "npm", ["view", "example@^1.0.0", "version", "--json"], @@ -2646,8 +2658,14 @@ export default function(api) { api.registerTool({ name: "test", description: "te .mockResolvedValue('["1.0.0","1.3.1","1.0.2"]'); const runCommandSpy = vi.spyOn(packageManager as any, "runCommand").mockResolvedValue(undefined); + const updated: string[] = []; + packageManager.setProgressCallback((event) => { + if (event.type === "result" && event.action === "update" && event.message) updated.push(event.message); + }); await packageManager.update("npm:example"); + expect(updated).toEqual([]); + expect(runCommandCaptureSpy).toHaveBeenCalledWith( "npm", ["view", "example@^1.0.0", "version", "--json"], @@ -2840,6 +2858,33 @@ export default function(api) { api.registerTool({ name: "test", description: "te expect(maxConcurrentGitUpdates).toBeGreaterThan(1); }); + it("should warn when an npm update leaves the installed version unchanged", async () => { + const stalePath = join(agentDir, "npm", "node_modules", "stale-pkg"); + const freshPath = join(agentDir, "npm", "node_modules", "fresh-pkg"); + mkdirSync(stalePath, { recursive: true }); + mkdirSync(freshPath, { recursive: true }); + writeFileSync(join(stalePath, "package.json"), JSON.stringify({ name: "stale-pkg", version: "1.0.0" })); + writeFileSync(join(freshPath, "package.json"), JSON.stringify({ name: "fresh-pkg", version: "1.0.0" })); + settingsManager.setPackages(["npm:stale-pkg", "npm:fresh-pkg"]); + + vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue('"1.0.1"'); + vi.spyOn(packageManager as any, "runCommand").mockImplementation(async () => { + // Simulate npm resolving stale metadata for stale-pkg only. + writeFileSync(join(freshPath, "package.json"), JSON.stringify({ name: "fresh-pkg", version: "1.0.1" })); + }); + + const events: ProgressEvent[] = []; + packageManager.setProgressCallback((event) => events.push(event)); + + await packageManager.update(); + + const warnings = events.filter((event) => event.type === "progress"); + expect(warnings).toHaveLength(1); + expect(warnings[0].source).toBe("npm:stale-pkg"); + expect(warnings[0].message).toContain("is still 1.0.0"); + expect(warnings[0].message).toContain("1.0.1"); + }); + it("should suggest npm source prefixes for update lookups", async () => { settingsManager.setProjectPackages(["npm:example"]); @@ -2913,6 +2958,81 @@ export default function(api) { api.registerTool({ name: "test", description: "te expect(result.extensions.some((r) => pathEndsWith(r.path, "extensions/old.ts"))).toBe(false); }); + // https://github.com/earendil-works/pi/issues/11375 + it("should refresh temporary git sources with a branch ref", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + const source = "git:github.com/example/repo@main"; + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "git") throw new Error("Expected git source"); + const installedPath = managerWithInternals.getGitInstallPath(parsed, "temporary"); + mkdirSync(join(installedPath, "extensions"), { recursive: true }); + writeFileSync(join(installedPath, "extensions", "index.ts"), "export default function() {};"); + + const updateGitSpy = vi.spyOn(packageManager as any, "updateGit").mockResolvedValue(undefined); + + const result = await packageManager.resolveExtensionSources([source], { temporary: true }); + expect(updateGitSpy).toHaveBeenCalledTimes(1); + expect(updateGitSpy.mock.calls[0][0]).toMatchObject({ ref: "main" }); + expect(result.extensions.some((r) => pathEndsWith(r.path, "extensions/index.ts") && r.enabled)).toBe(true); + }); + + it("should not refresh temporary git sources pinned to the checked-out commit", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + const source = "git:github.com/example/repo@abc1234"; + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "git") throw new Error("Expected git source"); + mkdirSync(managerWithInternals.getGitInstallPath(parsed, "temporary"), { recursive: true }); + + vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue( + "ABC1234def5678901234567890123456789012345\n", + ); + const updateGitSpy = vi.spyOn(packageManager as any, "updateGit").mockResolvedValue(undefined); + + await packageManager.resolveExtensionSources([source], { temporary: true }); + expect(updateGitSpy).not.toHaveBeenCalled(); + }); + + it("should refresh unpinned temporary npm sources when a newer version exists", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + const source = "npm:example"; + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "npm") throw new Error("Expected npm source"); + const installedPath = managerWithInternals.getNpmInstallPath(parsed, "temporary"); + mkdirSync(join(installedPath, "extensions"), { recursive: true }); + writeFileSync(join(installedPath, "package.json"), JSON.stringify({ name: "example", version: "1.0.0" })); + writeFileSync(join(installedPath, "extensions", "index.ts"), "export default function() {};"); + + vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue('"1.1.0"'); + const runNpmCommandSpy = vi + .spyOn(packageManager as any, "runNpmCommand") + .mockRejectedValue(new Error("network down")); + + const result = await packageManager.resolveExtensionSources([source], { temporary: true }); + expect(runNpmCommandSpy).toHaveBeenCalledTimes(1); + expect(runNpmCommandSpy.mock.calls[0][0]).toContain("example@latest"); + // A failed refresh keeps loading the cached install. + expect(result.extensions.some((r) => pathEndsWith(r.path, "extensions/index.ts") && r.enabled)).toBe(true); + }); + + it("should not refresh temporary npm sources when already up to date or pinned", async () => { + const managerWithInternals = packageManager as unknown as PackageManagerInternals; + for (const source of ["npm:example", "npm:example@1.0.0"]) { + const parsed = managerWithInternals.parseSource(source); + if (parsed.type !== "npm") throw new Error("Expected npm source"); + const installedPath = managerWithInternals.getNpmInstallPath(parsed, "temporary"); + mkdirSync(installedPath, { recursive: true }); + writeFileSync(join(installedPath, "package.json"), JSON.stringify({ name: "example", version: "1.0.0" })); + } + + const runCommandCaptureSpy = vi.spyOn(packageManager as any, "runCommandCapture").mockResolvedValue('"1.0.0"'); + const runNpmCommandSpy = vi.spyOn(packageManager as any, "runNpmCommand").mockResolvedValue(undefined); + + await packageManager.resolveExtensionSources(["npm:example"], { temporary: true }); + await packageManager.resolveExtensionSources(["npm:example@1.0.0"], { temporary: true }); + expect(runCommandCaptureSpy).toHaveBeenCalledTimes(1); + expect(runNpmCommandSpy).not.toHaveBeenCalled(); + }); + it("should not run npm view during resolve for installed unpinned packages", async () => { process.env.PI_OFFLINE = "1"; const installedPath = join(tempDir, ".pi", "npm", "node_modules", "example"); diff --git a/packages/coding-agent/test/paths.test.ts b/packages/coding-agent/test/paths.test.ts index fbdb4f56f..b57238c13 100644 --- a/packages/coding-agent/test/paths.test.ts +++ b/packages/coding-agent/test/paths.test.ts @@ -6,10 +6,14 @@ import { afterEach, describe, expect, it } from "vitest"; import { canonicalizePath, getCwdRelativePath, + getPathComparisonKey, isLocalPath, + isPathWithin, normalizePath, normalizeWindowsShellPath, + pathsEqual, resolvePath, + shortenHomePath, } from "../src/utils/paths.ts"; let tempDir: string; @@ -26,6 +30,64 @@ function createTempDir(): string { return tempDir; } +describe("path comparison", () => { + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + + afterEach(() => { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + }); + + function simulatePlatform(platform: NodeJS.Platform): void { + Object.defineProperty(process, "platform", { value: platform }); + } + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("treats drive letters and path components case-insensitively on Windows", () => { + simulatePlatform("win32"); + expect(getPathComparisonKey("c:\\Users\\me\\.agents\\skills")).toBe( + getPathComparisonKey("C:\\Users\\Me\\.agents\\skills"), + ); + expect(pathsEqual("c:\\Users\\me", "C:/Users/Me/")).toBe(true); + expect(pathsEqual("C:\\Users\\me", "D:\\Users\\me")).toBe(false); + expect(isPathWithin("C:\\Users\\Me", "c:\\users\\me\\project")).toBe(true); + expect(isPathWithin("C:\\Users\\Me", "C:\\Users\\Me2")).toBe(false); + expect(isPathWithin("C:\\", "c:\\Users")).toBe(true); + expect(isPathWithin("\\\\server\\share", "\\\\SERVER\\Share\\folder")).toBe(true); + }); + + it("keeps casing on case-sensitive platforms", () => { + simulatePlatform("linux"); + expect(pathsEqual("/home/me", "/home/Me")).toBe(false); + expect(isPathWithin("/home/me", "/home/Me/project")).toBe(false); + }); + + it("isPathWithin matches whole path segments", () => { + simulatePlatform("linux"); + expect(isPathWithin("/home/me", "/home/me")).toBe(true); + expect(isPathWithin("/home/me", "/home/me/project")).toBe(true); + expect(isPathWithin("/home/me/", "/home/me/project")).toBe(true); + expect(isPathWithin("/home/me", "/home/me2")).toBe(false); + expect(isPathWithin("/", "/home")).toBe(true); + }); + + it("shortenHomePath replaces only a leading home directory", () => { + simulatePlatform("linux"); + expect(shortenHomePath("/home/me", "/home/me")).toBe("~"); + expect(shortenHomePath("/home/me/project", "/home/me")).toBe("~/project"); + expect(shortenHomePath("/home/me2/project", "/home/me")).toBe("/home/me2/project"); + expect(shortenHomePath("/home/Me/project", "/home/me")).toBe("/home/Me/project"); + expect(shortenHomePath("/etc/passwd", "/")).toBe("~/etc/passwd"); + expect(shortenHomePath("home/me/project", "/home/me")).toBe("home/me/project"); + expect(shortenHomePath("/home/me/project", "")).toBe("/home/me/project"); + }); + + it("shortenHomePath ignores casing and keeps separators on Windows", () => { + simulatePlatform("win32"); + expect(shortenHomePath("c:\\users\\me\\project", "C:\\Users\\Me")).toBe("~\\project"); + expect(shortenHomePath("\\\\server\\share\\folder", "\\\\server\\share")).toBe("~\\folder"); + }); +}); + describe("canonicalizePath", () => { it("returns the real path for a regular file", () => { const dir = createTempDir(); diff --git a/packages/coding-agent/test/project-trust-saved-decision.test.ts b/packages/coding-agent/test/project-trust-saved-decision.test.ts new file mode 100644 index 000000000..f356e44a3 --- /dev/null +++ b/packages/coding-agent/test/project-trust-saved-decision.test.ts @@ -0,0 +1,150 @@ +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { createProjectTrustContext } from "../src/cli/project-trust.ts"; +import { ENV_AGENT_DIR } from "../src/config.ts"; +import { resolveProjectTrusted } from "../src/core/project-trust.ts"; +import { SettingsManager } from "../src/core/settings-manager.ts"; +import { ProjectTrustStore } from "../src/core/trust-manager.ts"; + +const cliPath = resolve(__dirname, "../src/cli.ts"); +// --import takes a module specifier, not a filesystem path. +const sourceResolverUrl = pathToFileURL(resolve(__dirname, "../src/experimental/source-resolver.ts")).href; +const tempDirs: string[] = []; + +afterEach(() => { + for (const dir of tempDirs.splice(0)) { + rmSync(dir, { recursive: true, force: true }); + } +}); + +interface Dirs { + agentDir: string; + projectDir: string; + probeFile: string; + extensionFile: string; +} + +function setup(): Dirs { + const tempRoot = realpathSync(mkdtempSync(join(tmpdir(), "pi-project-trust-saved-"))); + tempDirs.push(tempRoot); + const dirs = { + agentDir: join(tempRoot, "agent"), + projectDir: join(tempRoot, "project"), + probeFile: join(tempRoot, "probe.json"), + extensionFile: join(tempRoot, "probe-extension.ts"), + }; + mkdirSync(dirs.agentDir, { recursive: true }); + mkdirSync(dirs.projectDir, { recursive: true }); + // AGENTS.md alone is not a trust-requiring project resource. + writeFileSync(join(dirs.projectDir, "AGENTS.md"), "# untrusted\n"); + writeFileSync( + dirs.extensionFile, + `import { writeFileSync } from "node:fs"; +export default function (pi) { + pi.registerProvider("trust-probe", { + baseUrl: "https://trust-probe.invalid/v1", + apiKey: "trust-probe-key", + api: "openai-completions", + models: [{ + id: "trust-probe-model", + name: "trust-probe-model", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 10000, + maxTokens: 1000, + }], + }); + pi.on("session_start", (_event, ctx) => { + writeFileSync(${JSON.stringify(dirs.probeFile)}, JSON.stringify({ trusted: ctx.isProjectTrusted() })); + process.exit(0); + }); +} +`, + ); + return dirs; +} + +async function probeProjectTrust(dirs: Dirs): Promise { + const child = spawn( + process.execPath, + [ + "--import", + sourceResolverUrl, + cliPath, + "--no-session", + "-e", + dirs.extensionFile, + "--provider", + "trust-probe", + "--model", + "trust-probe-model", + "-p", + "hi", + ], + { + cwd: dirs.projectDir, + env: { ...process.env, [ENV_AGENT_DIR]: dirs.agentDir, PI_OFFLINE: "1" }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + const code = await new Promise((resolvePromise, reject) => { + const timeout = setTimeout(() => child.kill("SIGKILL"), 20_000); + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("close", (exitCode) => { + clearTimeout(timeout); + resolvePromise(exitCode); + }); + }); + expect(code, stderr).toBe(0); + return (JSON.parse(readFileSync(dirs.probeFile, "utf8")) as { trusted: boolean }).trusted; +} + +// Regression tests for https://github.com/earendil-works/pi/issues/10947 +describe("saved project trust for projects without trust-requiring resources", () => { + it("reports a saved Do not trust decision to extensions", async () => { + const dirs = setup(); + new ProjectTrustStore(dirs.agentDir).set(dirs.projectDir, false); + + expect(await probeProjectTrust(dirs)).toBe(false); + }, 30_000); + + it("auto-trusts when no decision is saved", async () => { + const dirs = setup(); + + expect(await probeProjectTrust(dirs)).toBe(true); + }, 30_000); + + it("resolveProjectTrusted honors the saved decision before auto-trust", async () => { + const dirs = setup(); + const trustStore = new ProjectTrustStore(dirs.agentDir); + const resolve = () => + resolveProjectTrusted({ + cwd: dirs.projectDir, + trustStore, + projectTrustContext: createProjectTrustContext({ + cwd: dirs.projectDir, + mode: "print", + settingsManager: SettingsManager.inMemory(), + hasUI: false, + }), + }); + + expect(await resolve()).toBe(true); + trustStore.set(dirs.projectDir, false); + expect(await resolve()).toBe(false); + trustStore.set(dirs.projectDir, true); + expect(await resolve()).toBe(true); + }); +}); diff --git a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts index 7c569f6e8..2813adc0f 100644 --- a/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts +++ b/packages/coding-agent/test/sdk-codex-cache-probe-tool-loop.ts @@ -21,8 +21,8 @@ import { getModel, type Model, type SimpleStreamOptions, - Type, } from "@earendil-works/pi-ai/compat"; +import { Type } from "typebox"; import { getOpenAICodexWebSocketDebugStats, streamSimple as streamSimpleOpenAICodexResponses, diff --git a/packages/coding-agent/test/source-info.test.ts b/packages/coding-agent/test/source-info.test.ts new file mode 100644 index 000000000..35e535704 --- /dev/null +++ b/packages/coding-agent/test/source-info.test.ts @@ -0,0 +1,19 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { getResourcePathKey } from "../src/core/source-info.ts"; + +describe("getResourcePathKey", () => { + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + + afterEach(() => { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + }); + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("folds file path casing on Windows but keeps synthetic identifiers unchanged", () => { + Object.defineProperty(process, "platform", { value: "win32" }); + expect(getResourcePathKey("C:\\Users\\Me\\ext.ts")).toBe(getResourcePathKey("c:\\users\\me\\ext.ts")); + expect(getResourcePathKey("builtin:MCP")).toBe("builtin:MCP"); + expect(getResourcePathKey("builtin:MCP")).not.toBe(getResourcePathKey("builtin:mcp")); + expect(getResourcePathKey("")).toBe(""); + }); +}); diff --git a/packages/coding-agent/test/suite/agent-session-codemode.test.ts b/packages/coding-agent/test/suite/agent-session-codemode.test.ts index ff39f1cd6..3e0620635 100644 --- a/packages/coding-agent/test/suite/agent-session-codemode.test.ts +++ b/packages/coding-agent/test/suite/agent-session-codemode.test.ts @@ -18,7 +18,10 @@ import { Type } from "typebox"; import { afterEach, describe, expect, it } from "vitest"; import type { ExtensionAPI } from "../../src/core/extensions/types.ts"; import type { CustomEntry } from "../../src/core/session-manager.ts"; -import { createToolDefinitionFromAgentTool } from "../../src/core/tools/tool-definition-wrapper.ts"; +import { + createToolDefinitionFromAgentTool, + type TypeBoxAgentTool, +} from "../../src/core/tools/tool-definition-wrapper.ts"; import { readCodemodeStore } from "../../src/extensions/codemode/execute.ts"; import { createCodemodeExtension } from "../../src/extensions/codemode/index.ts"; import { @@ -33,7 +36,7 @@ const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; const echoSchema = Type.Object({ text: Type.String({ description: "Text to echo" }) }); -const echoTool: AgentTool = { +const echoTool: TypeBoxAgentTool = { name: "echo", label: "Echo", description: "Echo text back.\n\nSecond paragraph.", diff --git a/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts b/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts new file mode 100644 index 000000000..02639a701 --- /dev/null +++ b/packages/coding-agent/test/suite/regressions/11520-sdk-summary-oauth-auth.test.ts @@ -0,0 +1,122 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + type Api, + createAssistantMessageEventStream, + type Model, + type SimpleStreamOptions, +} from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, it } from "vitest"; +import { AuthStorage } from "../../../src/core/auth-storage.ts"; +import { ModelRuntime } from "../../../src/core/model-runtime.ts"; +import { createAgentSession } from "../../../src/core/sdk.ts"; +import { SessionManager } from "../../../src/core/session-manager.ts"; +import { SettingsManager } from "../../../src/core/settings-manager.ts"; +import { assistantMsg, createTestResourceLoader, userMsg } from "../../utilities.ts"; + +// Issue #11520: summaries in SDK sessions passed the resolved OAuth token back to the +// ModelRuntime as an explicit API key, so provider auth took the API-key override branch. +describe("issue #11520 SDK summary OAuth auth", () => { + const tempDirs: string[] = []; + + afterEach(() => { + while (tempDirs.length > 0) rmSync(tempDirs.pop()!, { recursive: true, force: true }); + }); + + it("lets the runtime resolve stored OAuth for branch summaries", async () => { + const cwd = mkdtempSync(join(tmpdir(), "pi-11520-")); + tempDirs.push(cwd); + const credentials = AuthStorage.inMemory({ + "oauth-only-provider": { + type: "oauth", + access: "oauth-access", + refresh: "refresh", + expires: Date.now() + 3_600_000, + }, + }); + const modelRuntime = await ModelRuntime.create({ credentials, modelsPath: null, allowModelNetwork: false }); + const model: Model = { + id: "oauth-model", + name: "OAuth Model", + api: "openai-completions", + provider: "oauth-only-provider", + baseUrl: "https://example.test/v1", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; + const requests: SimpleStreamOptions[] = []; + modelRuntime.registerProvider(model.provider, { + name: "OAuth Only", + baseUrl: model.baseUrl, + api: model.api, + // With an API key method as well, an explicit key would bypass the stored OAuth login. + apiKey: "UNUSED_ENV_KEY", + oauth: { + name: "Subscription", + login: async () => ({ access: "oauth-access", refresh: "refresh", expires: Date.now() + 60_000 }), + refreshToken: async (credential) => credential, + getApiKey: (credential) => credential.access, + }, + models: [model], + streamSimple: (requestModel, _context, options) => { + requests.push(options ?? {}); + const stream = createAssistantMessageEventStream(); + stream.end({ + role: "assistant", + content: [{ type: "text", text: "branch summary text" }], + api: requestModel.api, + provider: requestModel.provider, + model: requestModel.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }); + return stream; + }, + }); + const authOverrides: Array = []; + const getAuth = modelRuntime.getAuth.bind(modelRuntime); + modelRuntime.getAuth = (async (...args: Parameters) => { + authOverrides.push(args[1]?.apiKey); + return getAuth(...args); + }) as ModelRuntime["getAuth"]; + + const sessionManager = SessionManager.inMemory(cwd); + const { session } = await createAgentSession({ + cwd, + agentDir: cwd, + model, + modelRuntime, + settingsManager: SettingsManager.inMemory(), + sessionManager, + resourceLoader: createTestResourceLoader(), + }); + try { + const targetId = sessionManager.appendMessage(userMsg("first branch")); + sessionManager.appendMessage(assistantMsg("first reply")); + sessionManager.appendMessage(userMsg("abandoned branch work")); + sessionManager.appendMessage(assistantMsg("abandoned reply")); + + const result = await session.navigateTree(targetId, { summarize: true }); + + expect(result.summaryEntry?.summary).toContain("branch summary text"); + expect(requests).toHaveLength(1); + expect(requests[0].apiKey).toBe("oauth-access"); + expect(authOverrides.length).toBeGreaterThan(0); + expect(authOverrides.every((apiKey) => apiKey === undefined)).toBe(true); + } finally { + session.dispose(); + } + }); +}); diff --git a/packages/coding-agent/test/system-prompt.test.ts b/packages/coding-agent/test/system-prompt.test.ts index 276cc63ff..86d2c6994 100644 --- a/packages/coding-agent/test/system-prompt.test.ts +++ b/packages/coding-agent/test/system-prompt.test.ts @@ -48,7 +48,9 @@ describe("buildSystemPrompt", () => { skills: [], }); - expect(defaultPrompt.startsWith("You are an expert coding assistant operating inside pi")).toBe(true); + expect(defaultPrompt.startsWith("You are a coding assistant running inside pi, a coding agent harness.")).toBe( + true, + ); expect(customPrompt.startsWith("You are Exact.\n\n")).toBe(true); }); @@ -119,6 +121,9 @@ describe("buildSystemPrompt", () => { "- When reading pi docs or examples, resolve docs/... under Additional docs and examples/... under Examples, not the current working directory", ); expect(prompt).toContain("environment variables (docs/environment-variables.md), MCP servers (docs/mcp.md)"); + expect(prompt).toContain( + "Before implementing harness changes, read the relevant docs and examples in full, following related cross-references.", + ); }); }); @@ -192,6 +197,9 @@ describe("buildSystemPrompt", () => { expect(prompt).toContain(""); expect(prompt).toContain("test-skill"); expect(prompt).toContain("Use bash to load a skill's file"); + expect(prompt).toContain( + "Resolve relative paths against the skill's directory (parent of SKILL.md). Use absolute paths in tool calls.", + ); }); test("omits skills without read or bash", () => { diff --git a/packages/coding-agent/test/tool-search.test.ts b/packages/coding-agent/test/tool-search.test.ts index fd55b5c56..68a59cec7 100644 --- a/packages/coding-agent/test/tool-search.test.ts +++ b/packages/coding-agent/test/tool-search.test.ts @@ -19,6 +19,40 @@ describe("tokenize", () => { it("splits camelCase and snake_case, drops stop words, and folds plurals", () => { expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]); expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]); + expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]); + }); + + it.each([ + ["搜索仓库问题", ["搜索", "仓库", "问题"]], + ["文書を検索", ["文書", "を", "検索"]], + ["문서 검색", ["문서", "검색"]], + ["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]], + ["Поиск документов", ["поиск", "документов"]], + ["البحث عن الملفات", ["البحث", "عن", "الملفات"]], + ["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]], + ["Éléments cafés", ["éléments", "cafés"]], + ])("segments Unicode words in %s", (text, expected) => { + expect(tokenize(text)).toEqual(expected); + }); + + it("preserves non-ASCII words and splits mixed-language identifiers", () => { + expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([ + "搜索", + "git", + "hub", + "repo", + "issue", + "résumé", + "cafés", + ]); + }); + + it("normalizes canonically equivalent Unicode text", () => { + expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]); + }); + + it("ignores whitespace, punctuation, and emoji", () => { + expect(tokenize(" \t，。!? 😀 ")).toEqual([]); }); }); @@ -45,6 +79,22 @@ describe("Bm25Ranker", () => { expect(ranker.rank("closed", documents, 8).map((match) => match.name)).toEqual(["mcp__github__list_issues"]); }); + it.each([ + ["搜索仓库问题", "问题"], + ["文書を検索", "検索"], + ["문서 검색", "검색"], + ["ค้นหาเอกสาร", "เอกสาร"], + ["Поиск документов", "документов"], + ["البحث عن الملفات", "الملفات"], + ["फ़ाइल खोजें", "खोजें"], + ["Rechercher un café", "cafe\u0301"], + ])("matches Unicode queries against %s", (description, query) => { + const localized = createToolSearchDocument(tool("localized", description)); + expect(new Bm25Ranker().rank(query, [...documents, localized], 8)).toEqual([ + { name: "localized", score: expect.any(Number) }, + ]); + }); + it("returns nothing for unknown or empty queries", () => { const ranker = new Bm25Ranker(); expect(ranker.rank("kubernetes", documents, 8)).toEqual([]); @@ -76,6 +126,64 @@ describe("Bm25Ranker", () => { expect(ranker.rank("read", named, 1).map((match) => match.name)).toEqual(["mcp__app__queue_read"]); }); + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a name match above mentions in descriptions", () => { + const documents = [ + tool("mcp__ci__status", "Deploy status: last deploy, deploy log, deploy errors."), + tool( + "mcp__ci__deploy_release", + "Start a release of the current branch to an environment, run the migrations, switch traffic over, and wait until the new version is live and healthy.", + ), + ].map((entry) => createToolSearchDocument(entry)); + expect(new Bm25Ranker().rank("deploy", documents, 1).map((match) => match.name)).toEqual([ + "mcp__ci__deploy_release", + ]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query contains its whole name", () => { + const peekaboo = { name: "mcp__peekaboo", instructions: "Peekaboo controls macOS apps by app name." }; + const documents = [ + tool("mcp__peekaboo__see", "Capture the app window of the peekaboo app and describe the app."), + tool("mcp__peekaboo__menu", "Click a menu item of an app."), + tool( + "mcp__peekaboo__app", + "Launch, quit, relaunch, focus, hide, or switch applications. Targets an application by name, bundle identifier, or process ID, and can wait until it is ready.", + ), + tool("mcp__app__queue_read", "Read a queued message of this thread."), + tool( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans.", + ), + ].map((entry) => + createToolSearchDocument(entry, entry.name.startsWith("mcp__peekaboo__") ? peekaboo : undefined), + ); + const ranker = new Bm25Ranker(); + const first = (query: string) => ranker.rank(query, documents, 1).map((match) => match.name); + expect(first("peekaboo app")).toEqual(["mcp__peekaboo__app"]); + expect(first("open the peekaboo app tool")).toEqual(["mcp__peekaboo__app"]); + expect(first("read thread_read state")).toEqual(["mcp__dev-app__thread_read"]); + // Without the whole name, mentions in descriptions decide. + expect(first("describe window")).toEqual(["mcp__peekaboo__see"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool matching more query terms first, whatever the scores", () => { + const documents = [ + tool("mcp__peekaboo__app", "Launch, quit, focus, or switch applications by name or bundle identifier."), + tool("mcp__peekaboo__see", "Capture the app window and describe the UI elements of the app."), + tool("mcp__peekaboo__window", "Move, resize, or close a window of an app."), + tool("mcp__peekaboo__image", "Take a screenshot of the screen, an app, or a window."), + ].map((entry) => createToolSearchDocument(entry)); + const ranker = new Bm25Ranker(); + const first = (query: string) => ranker.rank(query, documents, 1).map((match) => match.name); + // `window` and `app` are whole names, but each covers one of the two terms. + expect(first("screenshot window")).toEqual(["mcp__peekaboo__image"]); + expect(first("describe app")).toEqual(["mcp__peekaboo__see"]); + // An exact name still ranks first. + expect(first("window")).toEqual(["mcp__peekaboo__window"]); + }); + it("includes the namespace in the search text", () => { const document = createToolSearchDocument(tool("mcp__x__run", "Run it."), { name: "mcp__x", diff --git a/packages/coding-agent/test/trust-manager.test.ts b/packages/coding-agent/test/trust-manager.test.ts index 01500bac2..36d6f89ec 100644 --- a/packages/coding-agent/test/trust-manager.test.ts +++ b/packages/coding-agent/test/trust-manager.test.ts @@ -1,4 +1,4 @@ -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -64,4 +64,45 @@ describe("ProjectTrustStore", () => { } } }); + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("ignores the user .agents/skills dir when HOME differs only by case on Windows", () => { + const originalHome = process.env.HOME; + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + const mixedCaseHome = join(tempDir, "Home"); + const projectDir = join(mixedCaseHome, "project"); + mkdirSync(join(mixedCaseHome, ".agents", "skills"), { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + process.env.HOME = mixedCaseHome.toLowerCase(); + Object.defineProperty(process, "platform", { value: "win32" }); + try { + expect(hasTrustRequiringProjectResources(projectDir)).toBe(false); + } finally { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + } + }); + + // Regression test for https://github.com/earendil-works/pi/issues/10488 + it("matches saved decisions regardless of casing on Windows", () => { + const originalPlatform = Object.getOwnPropertyDescriptor(process, "platform"); + const projectDir = join(tempDir, "Project-Mixed"); + mkdirSync(projectDir, { recursive: true }); + const store = new ProjectTrustStore(agentDir); + store.set(projectDir, true); + Object.defineProperty(process, "platform", { value: "win32" }); + try { + expect(store.get(projectDir.toLowerCase())).toBe(true); + store.set(projectDir.toLowerCase(), false); + const saved = JSON.parse(readFileSync(join(agentDir, "trust.json"), "utf-8")) as Record; + expect(Object.values(saved)).toEqual([false]); + expect(store.get(projectDir)).toBe(false); + } finally { + if (originalPlatform) Object.defineProperty(process, "platform", originalPlatform); + } + }); }); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 56763148d..14750ec5a 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -2,10 +2,47 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + ### Breaking Changes +- Tool schemas are JSON Schema typed through `@earendil-works/pi-schema`: `defineTool()` and `ToolRegistration` take any JSON Schema object as `parameters` and `structuredOutputSchema`, and type `args` with pi-schema's `Static` (`ToolArgs`; TypeBox schemas give TypeBox's static type for the common kinds, as pi-schema's `Static` describes; other schemas give `unknown`). The built-in tools (`read`, `write`, `edit`, `bash`, `powershell`, `codemode`, `tool_search`, the model tools) are built with pi-schema, with the same JSON Schema up to key order. `structuredOutput` is validated with pi-schema, so `invalid_structured_output` messages use its wording. `typebox` is no longer a dependency. Migration: replace `Type.Object({ n: Type.Number() })` from `@earendil-works/pi-ai` with `obj({ n: num() })` from `@earendil-works/pi-schema` (closed by default; `{ open: true }` allows other properties). - `SqliteExecutor` (re-exported from Chord) gains `runCounted(sql, ...params)`, resolving with the rows the statement changed; custom `SqliteDatabase` implementations add it. -- `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption and publication of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption is published and fails the Session without rolling the adopted changes back. +- `Storage.commit()` stores a group of commits in one call: `commit(batches, receive, context)` resolves with a `CommitResult` (`{ stored, refused? }`) and replaces `commit(writes, adopt, context)`. Each batch is one atomic commit with its own seq, checked against the state the batches before it left. The first batch Durable's own checks refuse (global IDs, document actions, copy sources) is not stored, nor anything after it; the batches before it are, and the call resolves with their number in `stored` and the refusal in `refused`. Any other failure (a driver or I/O error, a failed `COMMIT`) rejects the whole call as an unknown outcome and stores nothing the backend could roll back. `receive(index, seq)` is called once per stored batch, in order, synchronously, once they are durable and before the next call starts; receipts made before a rejection stand. Every batch has at least one write. The optional `publishing(publish, context)` runs each commit's publication inside the storage's own scope (`StoreStorage`: the attached host's batch). Memory, SQLite, JSONL, IndexedDB and `StoreStorage` store a call in one transaction or append, and the storage conformance suite checks the contract. Migration for a custom Storage: + + ```ts + // Before + async commit(writes, adopt, context) { + const seq = await this.transaction((tx) => { + check(tx, writes); // throws when Durable's rules refuse the commit + return apply(tx, writes); + }); + adopt(seq); + return seq; + } + + // After + async commit(batches, receive, context) { + let refused: { error: unknown } | undefined; + const seqs = await this.transaction((tx) => { + const stored: Seq[] = []; + for (const writes of batches) { + try { + check(tx, writes); // only Durable's own checks; reads the earlier batches' writes + } catch (error) { + refused = { error }; + break; // keep the batches before it + } + stored.push(apply(tx, writes)); // a throw rejects the call and rolls all of it back + } + return stored; + }); + seqs.forEach((seq, index) => receive(index, seq)); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; + } + ``` + + A database error during the checks is not a refusal: it must reject the call. A wrapper forwards `commit()` and `publishing()`. Code that called `storage.commit(writes, adopt, context)` directly, such as a test seeding a storage, calls `commitWrites(storage, writes, context)` from `@earendil-works/pi-durable/testing` (one batch; resolves with its seq, throws the refusal). - `Storage.mintId()`'s contract is narrowed to IDs greater than every ID minted since the storage was opened and every ID stored, also across reopen. The bundled storages already behave so. - Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both, with `WatchChange` and `NativeWatchChange`). End reasons are `stopped`, `cancelled`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`); `retired` is gone (see below). An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. - Watch listeners take one change object before the context: `watch.start(async ({ value, ops, retired }, context) => …)` for document, conversation view (`Conversation.watch()`), and task graph watches. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`. @@ -98,11 +135,13 @@ - `RunFailureReason` (`pi.live.runFailure.reason`) gains `nothing_to_answer` and `turn_limit`, also an `unanswered` submission's `reason`: exhaustive switches over it need cases ([#11043](https://github.com/earendil-works/pi/issues/11043)) - `pi.live.run` has a required `turns` field, the run's tool rounds so far; code that writes `run` itself sets it (`turns: 0`). `pi.live` is version 3: its `migrate` adds `turns: 0` to a stored run without it, version 1 or a version 2 an earlier unreleased build wrote, whose run in flight otherwise failed its next tool round with "Value contains a non-finite number" ([#11043](https://github.com/earendil-works/pi/issues/11043)) +- A Session's close ends synchronous observers (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, `observeCommits()`) `failed` with the error "session closed" instead of `ended`, so an adapter following a Session (Chord's `self.follow`) fails and follows the next one; `ended` stays for an observed owner that ended (a task document's task), and watches still end `ended`. Before: `end: (end) => { if (end.kind === "ended") done(); }`. After: `end: (end) => { if (end.kind === "ended" || end.kind === "failed") done(); }`. + ### Added -- `StoreStorage` (`@earendil-works/pi-durable/storage/store`): Durable's `Storage` on Chord v2's store. `StoreStorage.open(store, owner, context)` keeps a Session's records as declared tables of the store under `owner` and its documents in the store's documents engine, takes its ids from the store's one allocator, and commits each batch as one write of the store's commit step, adopted in the batch of the host the store serves, so a Chord object and a Durable Session commit through one queue and one outbox. It passes the storage conformance suite; `close()` leaves the store open (the app owns it). +- `StoreStorage` (`@earendil-works/pi-durable/storage/store`): Durable's `Storage` on Chord v2's store. `StoreStorage.open(store, owner, context)` keeps a Session's records as declared tables of the store under `owner` and its documents in the store's documents engine, takes its ids from the store's one allocator, and stores each `commit()` call in one store transaction (`ChordStore.commitSeries`), one seq and outbox row per batch, publishing each commit in the batch of the host the store serves, so a Chord object and a Durable Session commit through one queue and one outbox. It passes the storage conformance suite; `close()` leaves the store open (the app owns it). - Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. -- Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). +- Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit when it is published after its storage call settled, and for a deferred one when memory adopts it, before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). - `ConversationTranscript` keeps what a client has of one conversation over any `ConversationView`: `merge(view)` returns the view with `items`, the client's whole transcript (entries, a `gap` where a later window did not continue it, and an `earlier` marker until the start is loaded), instead of the window; `loadEarlier(context)` and `fillGap(gap, context)` page entries through a loader such as `Conversation.entries()`, and `resume` is the window that continues it after a drop: `conversation.watch(context, transcript.resume)`. A merge costs O(items) and returns the same `items` array when it adds nothing; concurrent loads of the same page are shared ([#11327](https://github.com/earendil-works/pi/issues/11327)). - `pi.live.runFailure` (`RunFailure`, `RunFailureReason`) records why the newest run ended without an answer, `model_error`, `no_model`, `aborted`, `reset`, `faulted`, or `orphaned`, with the run's task and inputs and a `detail` where one exists (the error text, the missing model, the fault, the orphan reason). The commit that ends the run sets it; a later commit that starts a run removes it ([#11327](https://github.com/earendil-works/pi/issues/11327)). - `RunFailureReason` gains `interrupted` (the host stopped while the model was answering, with no retry left) and `restart_limit` (the run's task was cut off `settings.restartLimit` times in a row) ([#11139](https://github.com/earendil-works/pi/issues/11139)). @@ -115,11 +154,20 @@ - `HookApi.entry()` and `entries()`, and `TaskRuntime.entries()`: a hook or task reads the conversation's raw history, which compaction never changes, so a range bounded by entry IDs reads the same before and after a compaction and on a rerun ([#11141](https://github.com/earendil-works/pi/issues/11141)) - Run-end tasks: a `GenerationHooks.afterRun(run, api)` hook returns `{ tasks }`, built with the typed `runEndTask(task, input)`, after a run answered (a final answer or a `terminate` round; not when an `onYield` continuation is taken, not after a failure or an abort). The answer is committed and the run's inputs settle `done` first, so `Submission.wait()` resolves at the answer; the generation then creates the tasks as its children and waits for them, keeping `pi.live.run` (with `run.afterRun`) and the conversation busy, so idle waits include them, Stop aborts them, and a crash resumes them. Input queued meanwhile, steers included, is placed when they are terminal and starts the next run; a task admits a model-visible write or a fix-run follow-up in its own commit with `submit(tx, draft)`. A crash before the tasks exist asks the hooks again; a broken request (an uninstalled or built-in task, a non-JSON input, a throwing `initial()`) is reported and skipped. `run` names the run's entries: `first` (`pi.live.run.first`, carried into `runFailure.first` so a continue covers a failed run's history), `answer`, and `last` ([#11141](https://github.com/earendil-works/pi/issues/11141)) +- `registryReader(value, context)`: a `RegistryReader` over a Chord v2 registry value whose composition is a registry snapshot (`RegistryValue`): `snapshot()` reads the current composition, `subscribe` observes its compositions synchronously inside Chord's batch, so a Harness opened in a Chord object takes its extensions from a Chord registry. +- `GenerationHooks.beforeProviderRequest({ model, payload }, api)` replaces the provider request body of a generation request, for that request only, e.g. to set a provider-specific field. The body is in the provider API's format; the hook returns `{ payload }` or `undefined`. Hooks chain in extension order and run each time pi-ai builds a body; a throw is reported and the chain continues. Compaction requests do not use it ([#10804](https://github.com/earendil-works/pi/issues/10804)) + ### Changed - Document, conversation view, and task graph watches use Chord's watch handles (`Watch`). -- A stored commit is published when Storage adopts it, inside `Storage.commit()`: the commit's telemetry events and Durable's own observers (document watches, conversation views, the task graph, the scheduler, and waits) run inside `adopt`, so two views never show one commit apart. `subscribeCommits()` listeners run one microtask later, outside the storage's synchronous step, before every wait the commit resolves and every abort signal it fires; they have no order relative to watch listeners and `onWake`. With a storage that works after adopting (JSONL's sidecar reclaim), all of these run before `Storage.commit()` resolves. -- `subscribeCommits()` listeners are read when a commit's dispatch runs: a listener added between adoption and the dispatch gets the commit only when one was registered at adoption; one added after the dispatch ran misses it. +- Group commit: a Session no longer holds its line while a commit is stored. Commits that arrive while one storage call is in flight go out together in the next call, each with its own seq and its own outcome, so the conversations of one Session share storage round trips instead of queueing behind each other's. A job runs while earlier commits are being stored until it touches a key one of them wrote (a record, a document, a conversation's entries, a task's membership or cancellation state); then it waits until that commit is stored and published. Outcomes are delivered in line order, and every job after a commit that fails the Session gets `SessionFailed`. One Harness serving 128 conversations over storage with a 4 ms round trip went from about 5 to about 440 agent runs per second (median run 25 s to 0.28 s); with local SQLite or the Chord store, throughput at 128 conversations rises by about 30%, while with in-memory storage and no latency each commit costs somewhat more CPU than before (up to 25% fewer runs per second). At most 256 jobs wait for their outcome. See `docs/group-commit.md` and spec §4. +- A stored commit is published after its storage call settled, in line order, the commits of one call a macrotask apart (inside the storage's `publishing()` scope if it has one): the read caches and memory adopt it, then its telemetry events and Durable's own observers (document watches, conversation views, the task graph, the scheduler, and waits) run, so two views never show one commit apart. `observeCommits()` observers run one microtask later, before every wait the commit resolves and every abort signal it fires; they have no order relative to watch listeners and `onWake`. A call that settles after something else failed the Session publishes nothing; what it stored is durable and appears after reopen. +- A normal agent run makes no storage reads: the Session keeps recently read conversation, task and submission records (bounded by count, and by live work for tasks), the newest entry of each conversation, and the task documents of live tasks, and the scheduler keeps the context range of a running conversation. Read results (`tx.task()`, `tx.conversation()`, `getTask()`, `runtime.entry()`, published entries) are frozen objects shared with these caches; mutating one throws. +- A tool slot's `running` status and a nested call's final arguments, and a generation's attempt, are stored with the next commit, like streamed progress: a run with one tool call stores 12 commits instead of 14. After a crash a slot may show `pending` while its task's checkpoint is at execution; the rerun sets it again. +- A wait whose commit is still being stored when the Harness closes ends with the closed error; the commit is still stored and published before close completes. +- Cancellation is checked when a job starts, which can be earlier than before: an invocation-bound submit queued behind an abort mark that is not stored yet lands, and the abort's cascade then aborts its run. +- The scheduler's per-commit work no longer grows with the number of conversations: after a commit it re-checks idleness only for the conversations above the tasks that became live or ended, expires idle context ranges from an expiry-ordered index, and `wake()` takes the earliest sleep from the sleeping tasks and scans live tasks only while nothing runs. Before, every commit re-walked every conversation read in the last `contextRetentionMs` (default 10 minutes). One Session with 2048 conversations went from 100 to 805 agent runs per second (4096: 46 to 772; local SQLite at 2048: 88 to 459), with CPU per run flat at about 1.6 ms instead of growing linearly. +- `subscribeCommits()` listeners are read when a commit's dispatch runs: a listener added between publication and the dispatch gets the commit only when one was registered at adoption; one added after the dispatch ran misses it. - The abort signal of a newly marked running task fires after `subscribeCommits()` listeners instead of before them. - A throwing Durable commit observer fails the Session after `Storage.commit()` returned, after the other observers and the host listeners got the commit; the commit still stands. When Storage also rejects that commit after adopting it, the storage error fails the Session and the observer's error is reported as `listener_error`. @@ -135,7 +183,10 @@ - Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. - Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. - Fixed a host that crashes during a model request resending it forever, adding an aborted entry each time: the interrupted request is a failed attempt under the retry policy, and the run ends with `runFailure` reason `interrupted` once no retry is left; other work cut off `restartLimit` times ends instead of keeping its conversation busy ([#11139](https://github.com/earendil-works/pi/issues/11139)) +- Fixed the built-in `read` resolving to `""` for an image in Code Mode scripts, so `image(await tools.read(...))` threw: `read` declares a `structuredOutputSchema`, a text file's text or an image's `ImageContent`, as the coding agent's `read` does ([#11015](https://github.com/earendil-works/pi/issues/11015)) - Fixed the Code Mode extension loading pi-codemode's WASM QuickJS runtime (`quickjs-wasi`) although it only runs scripts in the sandbox the host passes: it imports `toCodemodeIdentifier` from `@earendil-works/pi-codemode/declarations` instead of `/portable` ([#11254](https://github.com/earendil-works/pi/issues/11254)) +- Fixed `tool_search` and Code Mode's `searchTools()` ranking a tool below tools that only mention its name, as the coding agent did before [#11440](https://github.com/earendil-works/pi/pull/11440) and [#11625](https://github.com/earendil-works/pi/pull/11625): `thread_read` with `limit: 1` returned `queue_read`, and `peekaboo app` missed `mcp__peekaboo__app`. Both now rank like the coding agent: a query that is exactly a tool's name ranks it first, a tool matching more of the query's words ranks above one matching fewer, a query word in the name outweighs any number of mentions in the description, and a tool whose whole name is in the query gets a large bonus. `searchTools()` also searches namespace instructions, as a low-weight field that does not lower a tool's score ([#11331](https://github.com/earendil-works/pi/issues/11331)) +- Fixed `tool_search` and Code Mode's `searchTools()` finding nothing for Chinese, Japanese, Thai, and other non-ASCII queries, because the tokenizer kept only ASCII letters and digits. Words in any script now match, text without spaces between words such as `搜索记录` is split into words, and canonically equivalent text such as `café` and `cafe\u0301` matches ([#10816](https://github.com/earendil-works/pi/issues/10816), [#11037](https://github.com/earendil-works/pi/pull/11037) by [@haoqixu](https://github.com/haoqixu)) ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/README.md b/packages/durable/README.md index d87c54d34..b9e54a068 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -257,15 +257,15 @@ registry.install(createCodingTools({ images: await createNodePhotonImages() })); A model without image input sees a placeholder instead of the image, and `read` says so in a diagnostic. `/images/node` prepares images in a worker thread, so a large photo (a second or two of work) never blocks the event loop; the worker exits after 30 s without images, freeing Photon's WebAssembly memory (`createNodePhotonImages({ worker: false, idleMs })`). Elsewhere, `createPhotonImages()` and `/images/cloudflare` work on the calling thread, and decoding takes the image's full size in WebAssembly memory, which does not shrink again; leave the processor out on memory-constrained hosts such as Cloudflare Workers unless images stay small. -Define your own tool with a TypeBox schema. `defineTool()` types `args` from `parameters`, which the Harness validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: +Define your own tool with a JSON Schema, built here with `@earendil-works/pi-schema`. `defineTool()` types `args` from `parameters` (through pi-schema's `Static`; other JSON Schema gives `unknown`), which the Harness coerces and validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: ```typescript -import { Type } from "@earendil-works/pi-ai"; +import { num, obj, str } from "@earendil-works/pi-schema"; const count = defineTool({ name: "count", description: "Count from 1 to n", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (args, api) => { for (let i = 1; i <= args.n; i++) api.output(`${i}\n`); return {}; @@ -282,8 +282,8 @@ A result has three channels: `output` for the model, `structuredOutput` for prog const lines = defineTool({ name: "lines", description: "Count the lines of a file", - parameters: Type.Object({ path: Type.String() }), - structuredOutputSchema: Type.Object({ lines: Type.Number() }), + parameters: obj({ path: str() }), + structuredOutputSchema: obj({ lines: num() }), execute: async (args, api, context) => { const text = await readText(api, args.path, context); const count = text.split("\n").length; @@ -292,7 +292,7 @@ const lines = defineTool({ }); ``` -A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, even when an `afterTool` hook returns one, and their `diagnostics` say what went wrong; one a hook turns into a success gives a schema-less tool's program its output. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. +A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it, and programs then find its text in a `tool_error` diagnostic; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, even when an `afterTool` hook returns one, and their `diagnostics` say what went wrong; one a hook turns into a success gives a schema-less tool's program its output. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. The built-in `read` returns a text file's text as a string and an image as its `ImageContent`, which code mode's `image()` shows. A later extension's tool with the same name replaces an earlier one where both are selected, and `wrapTool()` decorates whichever tool won: @@ -312,7 +312,7 @@ A tool calls another tool with `api.executeTool()`, as a code mode script or an const testsPass = defineTool({ name: "tests_pass", description: "Run the test suite and say whether it passed", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, context) => { const run = await api.executeTool("bash", { command: "npm test" }, context, { progress: false }); // Absent when bash did not finish, for example on a timeout. @@ -659,7 +659,7 @@ const Guard = defineExtension({ }); ``` -- **Generation:** `beforeRequest` (replace the messages of one request), `afterResponse`, `onYield` (continue the run with another user message), `afterTools` (runs once a round's tools are done), and `afterRun` (tasks to run before the run ends, below). +- **Generation:** `beforeRequest` (replace the messages of one request), `beforeProviderRequest` (replace the provider request body of one request, e.g. to set a provider-specific field), `afterResponse`, `onYield` (continue the run with another user message), `afterTools` (runs once a round's tools are done), and `afterRun` (tasks to run before the run ends, below). - **Tools:** `beforeTool` (block or rewrite arguments) and `afterTool` (replace the result). `afterTool` sees every result a tool task writes, also the error results the Harness writes itself (unavailable, invalid, blocked, interrupted, restart limit, aborted), told apart by their diagnostic code; past `settings.restartLimit` it no longer runs. On an abort it runs inside the abort handler, so a slow hook delays `abort()`. To limit a hook to some conversations, select its extension only there, for example with `configure({ extensions: { add: [Guard] } })`. @@ -722,7 +722,7 @@ const Subagent: Extension = defineExtension({ defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String() }), + parameters: obj({ task: str() }), replay: "safe", // a rerun after a crash finds the same child and submission execute: async (args, api, context) => { const child = await api.commit(async (tx) => { @@ -889,7 +889,7 @@ await database.transaction(async (transaction) => { }); ``` -A custom backend's `commit(writes, adopt, context)` calls `adopt(seq)` once the batch is durable, synchronously and before its next commit starts, and then resolves with `seq`; a commit that fails before the batch is durable never calls it. `adopt` is the Session's in-memory adoption and publication of the batch, so memory and every view never show a partial log: inside it the Session's own observers run (views, task graph, scheduler, waits), and `subscribeCommits()` listeners, watch listeners and the code the commit wakes follow in microtasks, so at the backend's next `await` if it still works after `adopt` (as JSONL reclaims sidecars), before its `commit` resolves. `adopt` calls no Storage method and never throws; once it ran, the commit is published even if the backend then rejects, and the Session then fails. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: +A custom backend's `commit(batches, receive, context)` stores `batches` in order, each as one atomic commit with its own seq, checked against the state the earlier ones left, in as few round trips as it can. It stops at the first batch its own checks refuse (global IDs, document actions, copy resolution): the batches before it stay stored, it and the later ones are not. Once they are durable it calls `receive(index, seq)` per stored batch, in order and synchronously, then resolves with `{ stored, refused? }`; it rejects for an unknown outcome (connection lost, a failed statement or `COMMIT`). `receive` is a durability receipt: the Session publishes each batch later, one at a time, so memory and every view never show a partial log. `receive` calls no Storage method and never throws. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: ```typescript import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; @@ -905,7 +905,7 @@ registerStorageConformance({ describe, expect, it }, "My Storage", async (use) = }); ``` -The package root loads TypeBox, because the tool task validates arguments with pi-ai's `validateToolArguments()`. That costs about 23 MB of peak RSS unbundled, about 4 MB in a tree-shaken bundle. +The package root loads pi-schema, because the tool task validates arguments with pi-ai's `validateToolArguments()`. ## Errors diff --git a/packages/durable/docs/group-commit.md b/packages/durable/docs/group-commit.md new file mode 100644 index 000000000..4af775d71 --- /dev/null +++ b/packages/durable/docs/group-commit.md @@ -0,0 +1,296 @@ +# Group commit for the Durable Session + +Status: design, revision 2 (after three reviews). Branch `durable-perf` (worktree `~/pi-perf` on slayer), on top of the +uncommitted prerequisites (record cache, conversation tails, kept context ranges advanced at publish, task-document +index, deferred tool-slot `running` and generation attempt). Read `docs/spec.md` §1, §4, §4.1, §5, §8, §10.1 alongside. + +## 1. Problem + +A Session has one mutation line (`SessionImpl.#tail`). Each commit runs its callback, settles, calls `Storage.commit()`, +and adopts while holding the line, so every storage round trip of every conversation is serialized. After the +prerequisites the line makes no storage reads in a normal run; a run that calls one tool stores 12 commits. Measured on +slayer (one Harness, faux model, MemoryStorage with a simulated round trip per call, 32 concurrent conversations): +26.8 runs/s at 4 ms, 84 at 1 ms, 895 at 0 ms (CPU, one core). Throughput is fixed by the line. + +Group commit sends the commits that accumulate while one write is in flight as one storage call. Modelled with result +fences (each job on a run's critical path waits about one round trip): ~250 runs/s at 4 ms with 32 conversations, +rising toward the CPU ceiling with more conversations. A conflict probe (`/tmp/conflict`) measured, per run, 0 writing +jobs that would wait for one tool call or a streamed answer, 0.13 for three parallel tools. + +## 2. Semantics + +### Unchanged + +- Every logical commit keeps its own `seq`; history (`snapshotAsOf`, fork `asOf` copies, `createdAt`/`retiredAt`) + keeps per-commit boundaries. +- A commit's promise resolves only once it is durable and published; outcomes are delivered in line order (§5.1). +- Publication is in line order, deferred commits included (§5.2): telemetry events, Session listeners, the + `observeCommits()` dispatch, waits and aborts per commit; host observers of commit A run, and the continuations of + public waits A resolves start, before commit B is published (§5.3). +- No callback observes unstored state (§4). Memory adopts a commit at its publication. +- A commit adopted by Storage and then rejected stays published and rejects, as today (§3.4). +- Storage failure is fatal; callers are never told something storage does not hold. + +### Changed (accepted) + +1. A callback can run while an earlier commit's storage call is in flight. It runs until its first touch of a key an + unpublished commit wrote, waits until that commit's storage call settled and it is published, then continues. Code + before that touch runs early, assignments to caller variables included; acting on such values before the commit + resolves was already unsafe (a callback's own writes can fail to store today). +2. Cancellation is checked when a job starts; a job can start earlier. +3. If a batch fails fatally, later callbacks may already have run; their outcomes are `SessionFailed` (§5.1). +4. On a path that fails the Session, which commits survive differs: a stored prefix survives. Every caller's outcome + matches storage. On a bug path (adoption or an internal listener of part i throws) parts after i can be durable but + unpublished; their commits reject with `SessionFailed`; reopening recovers them. Likewise, once another error has + failed the Session, a call that settles afterwards, and the parts of a call not yet published when it fails, are + durable but unpublished and reject with `SessionFailed` (the first error); an `observeCommits()` consumer, such as a + replica, misses them until reopen. +5. Read results (`tx.task()`, `tx.conversation()`, `getTask()`, `runtime.entry()`, entries in published commits) are + frozen shared objects; mutating one throws (documented on `Tx`). +6. Prerequisite presentation changes (spec §8.2, §8.4 updated): a tool slot's `running` and a nested slot's final + arguments, and the generation's attempt, are stored like progress; after a crash a slot may show `pending` while + the task's checkpoint is `execute` (a safe rerun sets it again). +7. A wait whose commit is in flight when close begins ends with the closed error; the commit is still stored and + published by the drain (today this already happens when close lands before adopt). +8. Spec text to update: invariant 7, §4 (host listeners run while the line is held), §4.1 (holds leave at the last step + before `Storage.commit()`: now at logical admission), §10.1 (per-part outcomes), the `observeCommits` doc, the + `SessionImpl` class doc, the scheduler class invariant (decisions stay serialized; their storage overlaps). + +## 3. Storage + +### 3.1 `commit` + +```ts +/** + * Persist `batches` in order, each as one ordinary atomic commit with its own seq, checked against the state the + * earlier ones left, in as few round trips as the backend can. Stops at the first batch Durable's own checks refuse: + * the batches before it are durable, it and the ones after are not. Calls `receive(index, seq)` once per durable batch, + * in order, synchronously, after all of them are durable and before the next commit starts. Resolves with the number of + * durable batches and the refusal, if any. Rejects for an unknown outcome (connection lost, statement or commit + * failed); `receive` calls made before the rejection still stand. + */ +commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, +): Promise<{ readonly stored: number; readonly refused?: unknown }>; +``` + +`receive` is a durability receipt, not publication: the Session publishes later (§5.3). It is Storage's only write +method (it replaced the earlier one-batch `commit(writes, adopt, context)`, and took its name): every backend implements +it natively. Tests and suites that write one batch directly use `commitWrites()` (`@earendil-works/pi-durable/testing`), +which throws the refusal. + +Example: `[A, B, C, D]`, C refused. A and B are durable and published; C rejects with the refusal; D rejects with +`SessionFailed`; the Session fails. This is today's outcome (A and B stored, C fails, D never reaches storage). + +### 3.2 Backends + +A refusal is one of Durable's own checks (`checkGlobalIds`, `checkDocumentActions`, copy resolution), run against the +transaction's state before that batch writes anything. Any other error is an unknown outcome. No savepoints (Durable +Objects' SQL API does not allow them). + +- SQLite: one transaction; per batch: read `next_seq`, run the checks, apply, advance; on a refusal stop and commit the + prefix. `prepareDocumentActions` and `candidateNextId` per batch. +- MemoryStorage: per batch `prepareCommit()` (checks and detaches, changing nothing) then apply; stop at the first + refusal (prefix semantics allow it). +- JSONL: check batch by batch against the in-memory state, append the prefix's commit markers in one write; a torn append + recovers a shorter complete prefix, which prefix semantics allow. +- IndexedDB: one `readwrite` transaction per call (metadata read once, written once at the end). Per batch every read + and check runs first (`checkGlobalIds`, `checkDocumentActions`, then `resolveCopies`, which reads each copy's source + against the state the earlier batches left), and only then its writes. A refusal stops issuing writes; the transaction + commits the prefix. An error of IndexedDB itself (a `DOMException`, or a failed request or transaction without an + error, also as a check's cause) aborts the transaction: the call rejects with nothing stored. +- StoreStorage (p3's primary storage): one series of the Chord store's commit queue per call + (`ChordStore.commitSeries`): one backend transaction for all batches, one seq and one outbox commit per batch, the + store's metadata (`last_seq`, `next_id`) written once per call. A batch's `DurableCommit.check(step)` (ids, + documents, copies through the step's reader, so it sees the earlier batches; the same reads a single commit makes) + runs before its writes; a throw refuses it and the series commits the prefix, unless one of the check's statements + failed (the store watches the check's executor), which rolls the series back and rejects, as does a throw of the + writes; a failed `COMMIT` is an unknown outcome that stops the store. Receipts run after `COMMIT`, before the next + step of the queue; the Session then publishes each commit in its own host batch (`publishing`). Memory per call is + the call's staged commits and one receipt entry per batch; the catalog is copied only if a batch changes a table + record. + +A Storage may also implement `publishing?(publish: () => void, context: Context): void`: the Session calls it for each +part's publication and the Storage runs `publish` inside its own scope. StoreStorage implements it with its attached +host's `batch.within(...)`, so Chord sources over Durable's views stay marked in one batch per commit. + +### 3.3 Wrappers + +- `CachedStorage` applies write-through at publication (§5.3), not at the receipt. Its read rule becomes per key: a read + records `version(key)` at its start and keeps its result only if no commit published since wrote that key. Versions + exist only for keys with a read in flight (registered at read start, removed at read end), so the map is bounded by + reads in flight. +- `GuardedStorage.commit` guards the call but does not fail the Session itself on rejection; the writer does + (§3.4), after publishing the received parts. + +### 3.4 Outcomes per logical commit + +The writer keeps a ledger per call: which parts were received. + +- Received: published (§5.3), then resolved through the fences. +- `commit` resolved with a refusal at part k: parts before k as received; part k rejects with the refusal; parts + after k and every queued job: `SessionFailed`; the Session fails after the received parts are published. +- `commit` rejected: received parts are published first, then reject with the error (adopted-then-rejected, as + today); unreceived parts reject with the error; the Session fails. +- Adoption or an internal listener of part i throws: part i as today (the Session fails after its storage call); + parts after i are not published and reject with `SessionFailed`. + +## 4. The line + +### 4.1 Mechanics + +- The line runs one job at a time; callbacks never interleave with each other's synchronous code. +- A job with writes, after its callback and settlement succeed, takes the held changes it will store (`takeHeld`) and + joins the pending batch (logical admission). Its outcome is parked (§5.1). +- The writer runs off the line: it sends the pending batch with `commit` when no call is in flight and the previous + call's parts are published. A job waiting on a gate never blocks the writer. +- Backpressure: at most `MAX_OUTSTANDING` (initially 256) admitted jobs whose outcome is not delivered, writing or not; + beyond it the line waits. +- Unpublished write set: per key, the newest unpublished writer. Registered at logical admission (immediate commits) or + at callback end (deferred commits, which stay in it until their publication); removed at publication only if the key + still maps to that writer. +- Touch: every read or write a job makes checks its key. On a hit the job awaits the gate of the newest writer of that + key, then proceeds. A gate opens when that commit is published and its storage call settled (holds released). + +### 4.2 Keys + +| Key | Read by | Written by | +|---|---|---| +| `task:` | `tx.task`, committed-task reads, every task record a scheduler region reads (decided task, wait members and their `on`, cycle walks) | task create or write | +| `owned:` | every node a walk over live children expands (`#hasOwnedLive`, `#ownedLive`, `#walkDown`, `#waitCycle`, `ownedTasks`, `#idle` in a line job) | task creation and a task becoming terminal: every node from the task's parent up to and including the nearest live task or the root conversation (the nodes `#relist` lists or unlists), chain loaded at admission if needed; conversation creation under a task: that task's chain | +| `roots` | global idle in a line job (`#idle(undefined)`) | every task creation whose chain reaches an ownerless conversation (directly or through terminal tasks and conversations), and every task becoming terminal there (conservative: whether the root is listed can change under pending commits) | +| `intent` | every cancellation check (`#belowCancelled`, `#cancellingOwner`, including the empty-intent fast path, `#inScope`, `#nearestLiveAbove`) | abort marks (set, upgraded, cleared), a non-completed held or terminal outcome, restart marks | +| `conv:` | `tx.conversation`, `#requireConversation`, edge loads | conversation creation | +| `entry:` | `tx.entry(id)`, `storage.entry(conv, id)` | entry append | +| `entry-append:` | `latestEntryId`, `latestHeadMarker`, entry scans, context bounds | entry append (blind) | +| `sub:` | `tx.submission`, settlement/placement resolution | submission create or write | +| `req::` | `submissionByRequest` | submission creation with a request ID | +| `doc:
` | `tx.doc`, `retireDoc`, cold loads, `conversationDocumentOnLine`, fork copies of the source | document create/change/retire/copy at the address; held changes the commit takes | +| `docscope:` | `scanDocuments`, task-document index reads | document creation/retirement in the scope (blind) | +| `*` | scans that cannot be keyed (`scanTasks`, `scanSubmissions` by status, `scanConversations`); `unloadDocuments`, close, version reload | conflicts with every unpublished write | + +Blind writes conflict with reads of the key, not with each other (appends with distinct minted IDs, creations in one +scope). Created records' keys are ordinary keys: a minted ID can leak through a caller variable before publication, and a +read of it waits. + +### 4.3 Liveness-only reads (not keyed) + +A stale read that only delays work needs no key: a reservation pass not seeing a task a pending commit made runnable; +a waiter's check whether its dependencies are still live; the projection of immutable ownership fields (`owner`, +`conversationId`, `background`) during chain loading (not the rest of `#loadChain`, which installs settled nodes and +null edges after the check); a reservation pass skipping a task with an invocation without reading its record. +Cancellation decisions and terminalization are never liveness-only. + +### 4.4 Line reads outside `tx` + +`readOnLine(job)` becomes `readOnLine((storage) => job)`: the job gets a gated Storage view whose reads record keys and +wait on gates. All current `readOnLine` call sites (harness.ts, scheduler.ts runtime reads, submissions.ts, task-graph.ts, +view.ts, `captureContextBounds`) use it; the Session's line helpers (`latestEntryId`, `latestHeadMarker`, +`conversationDocumentOnLine`, cold loads) are gated too. Off-line reads keep using the ungated Storage. + +## 5. Outcomes and publication + +### 5.1 Outcome fences + +Every job's outcome (commits with and without writes, `readOnLine` jobs, deferred commits, callback errors) is +delivered in line order. A Session-fatal failure makes every later outcome `SessionFailed`; a nonfatal rejection (a +callback's own error, cancellation, validation, `ReadOnlySession`) affects only its job. Fences hold outcomes, not the +line. Exemption: a scheduler-internal job without writes whose outcome goes only to scheduler code and whose touch set +is clean (no hit) delivers at once; it is equivalent to having run before every unpublished commit. A job with writes +(a reservation, a step that faults, a reconcile pass) is never exempt: its outcome waits for its own publication, so no +invocation starts on a reservation that is not durable. + +### 5.2 Deferred commits + +A deferred commit's callback runs and settles like any job; its memory adoption and publication are fenced like an +immediate commit's publication: they happen when every earlier unpublished commit is published. Its keys stay in the +unpublished write set until then. Holds are taken at logical admission, so a commit never stores a hold adopted after +it in line order. + +### 5.3 Publishing a call's parts + +After `commit` settles (or the received prefix is known, §3.4): publish part 0 (cache write-through, conversation +tails, adoption, Session listeners, host dispatch, waits), then a macrotask hop (`setImmediate` where available, else a +`MessageChannel` message), then part 1, and so on. The hop lets public wait continuations (`Submission.wait()`, +`waitForTask()`, `commit()`) of part i start before part i+1 is published. Tests use the public APIs. + +### 5.4 Close, unload, holds, eviction + +- Close: seal admission; run close listeners; signal every invocation (scheduler); let admitted callbacks finish, + gates included; drain pending and in-flight calls and their publication; join invocations and run `writeAtClose()` + through the writer; store remaining holds through the writer; end observers; close Storage. +- `unloadDocuments()`, version reloads (also nested inside a callback), hold timer stores: `*` barriers through the + writer; holds captured after the barrier. +- Documents with unpublished changes stay pinned. + +## 6. Scheduler + +Scheduler state changes at publication, as today. Inside a callback, decisions read it synchronously. Each decision is +restructured as a pure plan: + +1. A decision region runs with a synchronous "recording" flag set; helpers record keys (§4.2) only while it is set + (the same helpers run at publication inside `#observe`, unrecorded). +2. The region reads, records, and computes a plan or an error, staging nothing and mutating no scheduler state. +3. If a recorded key has an unpublished writer: discard the plan (or error), await the gate, reload scopes + (`#loadScopes`), rebuild the overlay (`overlayOf`, `#links`), and run the region again. A waiting job holds the line, + so unpublished writes only shrink; re-runs converge. +4. After a clean check: apply the plan: stage writes, mutate scheduler sets, create or end invocations, run `migrate`, + take registry snapshots, report. None of these runs inside a region, so none repeats. + +Sites to restructure (hardest first): `#reserve` (runnable iteration, abandoned marks, cut-off faults, `#resolve` with +`migrate` and reports, invocation creation); `#step`/`#decide` (registry refresh, `task_outdated` reports, +cancellation, `#end`); `#finalize` (rounds, overlay mutation, awaited `#settleOutcome` between rounds: one region per +round); `#validateWait`/`#waitCycle` (async member loads, cycle errors gated like plans); `abort`; `#commitState` +terminal (chain loads per overlay task, walk); `#reconcile` (intent walks, marks; it captures and clears work lists +before its commit: move the take inside, after the check). `abortConversation` and cascades scan with `*`. + +Sweep: `#sweep` and `#scheduleSweep` do not drop nodes loaded by a job whose commit is unpublished; chain loads record +the nodes they install for their job, released at that commit's publication or failure. + +Later, the ownership index can move into the Session behind `tx` reads with the same keys. + +## 7. Bounds + +- Outstanding jobs: at most `MAX_OUTSTANDING`, a count, not bytes: pending writes are bounded by what one commit can + hold (tool output is bounded by output limits), times the count. +- Unpublished write set: one entry per key per newest writer, removed at publication. +- Cache versions: only keys with a read in flight. +- Fences, gates, ledgers: settled at publication or Session failure; nothing outlives close. +- Existing: record cache (conversations 4096, submissions 256; tasks: every live one, the ended ones a waiting or just-resumed task names, plus 256 ended), conversation tails (4096 IDs, 256 markers), + task-document index (live tasks and their documents), kept context ranges (busy or within retention; an idle range + past its expiry is dropped when an entry would extend it; expiry fixed at idle start, the timer uses the current + setting: document and test), document cache (retention). + +## 8. Tests + +- `commit` per backend (conformance for every backend): prefix stored, refusal index, seq per batch, copies + against earlier batches' state, JSONL torn append; IndexedDB: a copy refused mid-call keeps the prefix, an IndexedDB + error in a check or a write rejects with nothing stored; StoreStorage: one backend transaction per call; Chord's + `commitSeries`: refusal, a failed check statement and a throwing write roll back, a throwing receipt stops the store. +- Gating per key class, with deterministic interleavings: records; ancestor chain membership (live P → terminal B → + terminal A → empty C, creation in C vs P terminal); global idle with empty roots; pending abort mark vs child step + (empty-intent fast path); concurrent opposite waits; leaked entry ID read by `tx.entry`; request IDs; absent + addresses; held documents (hold created after a reader prepared); `*` barriers; `readOnLine` read after an + un-awaited commit. +- Outcome fences: no-write, read, deferred, callback errors in line order; nonfatal rejections do not poison later jobs; + fatal failure poisons later outcomes. +- Deferred publication behind a pending immediate commit, and when that commit fails. +- Failures per part: refusal, rejection after receipts (received parts published, then rejected), adoption failure. +- Publication order through public waits: A's `Submission.wait()`/`waitForTask()` continuation sees B unpublished. +- Close during in-flight calls with an invocation waiting on its signal; unload, version reload, hold timer during + flight; backpressure on all outstanding jobs. +- Scheduler retries: `migrate`, registry snapshots, reports and cleanup run once across re-runs; sweep vs chain loads. +- Randomized interleavings: random commits over overlapping keys, random latency and refusals; storage equals the stored + commits replayed in seq order, no callback observed unpublished state (instrumented oracle), outcomes match storage. +- Prerequisites: tool crash at both sides of the intent commit (sibling flushing presentation first); kept range + installed after a publication between return and install; live retention change; cache eviction and close; tails + eviction, fork markers. +- Bounds: outstanding, write sets, versions, fences return to zero when idle. +- Existing suites: durable (both retention modes), workers. + +## 9. Benchmark + +Conc probe at 0/1/4/20 ms with 1/8/32/64/128 conversations, one tool, three parallel tools, streaming; dpetrou's repro +as is and with warm documents. Report throughput, p50/p95 run latency, batch sizes, gates per run, fence delay. diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 4f3795e33..8f6b04982 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -65,11 +65,15 @@ The core rule is: Required invariants: 1. One Session commit is atomic across all record and document writes. -2. A stored commit is published once Storage made it durable, when Storage - adopts it (section 4), also when `Storage.commit()` then rejects; a commit - that fails before it is durable publishes nothing. A deferred commit - (section 4.1) is published to watches and the conversation view when memory - adopts it, and to `observeCommits()` once it is stored. +2. A stored commit is published once Storage made it durable, after the + storage call that stored it settled, in line order (section 4), also when + that call then rejects; a commit that fails before it is durable publishes + nothing. Once the Session has failed, nothing more is published: a commit + whose call settles after another error failed the Session, or whose turn to + be published comes after that, is durable but unpublished, and reopening + recovers it (section 10.1). A deferred commit (section 4.1) is published to watches and the + conversation view when memory adopts it, once the commits before it are + published, and to `observeCommits()` once it is stored. 3. All visible progress is durable, except what a deferred commit holds: what watches see can be ahead of Storage by up to that commit's window (`withinMs`), and a crash loses it. A commit is deferred only when its caller @@ -81,15 +85,16 @@ Required invariants: 6. Document drafts are fully revoked when their transaction callback settles: the Session synchronously prepares or aborts every open change at that point. Values assigned into a draft are copied by value and must be strict JSON. -7. The mutation line remains held through storage settlement and committed-state - adoption. Commit/close observers run synchronously only to capture immutable - state; document-state and watch user callbacks run later, as microtasks the - line never waits for, and never inside Storage's synchronous `adopt`. With a - storage that works after `adopt` (JSONL), they may run while the line is - still held by the commit that published. +7. The mutation line runs one commit callback at a time; their storage + overlaps (group commit, section 4). A callback never observes what an + unpublished commit wrote: at its first read or write of such a key it waits + until that commit is stored and published. Commits are published in line + order, and every outcome is delivered in line order. Commit/close observers + run synchronously only to capture immutable state; document-state and watch + user callbacks run later, as microtasks the line never waits for. 8. A storage error is fatal to the open Session (section 10.1), which must be reopened. A commit it fails publishes nothing, unless Storage had already - made it durable and adopted it. Preparation and checkpoint failures occur before + made it durable (a durability receipt). Preparation and checkpoint failures occur before storage admission and roll back normally. ## 2. Core records @@ -1607,24 +1612,25 @@ callback fails with no pending acquisition callback succeeds with no pending acquisition prepare every open change -> immutable next revision + self-contained Chord Op[] Session evaluates each required/ordinary document write exactly once - Storage.commit persists the atomic batch while the Session line remains held commit deferred and holdable (section 4.1) - adopt every prepared change by pointer swap; queue its ops per document, unstored - watches and view mounts capture it synchronously; observeCommits() does not - release the line; invoke listeners later -otherwise, right before Storage admission + join the writer's queue unstored; release the line + once the commits before it are published: adopt every prepared change by pointer swap; + queue its ops per document; watches and view mounts capture it synchronously; + observeCommits() does not; invoke listeners later +otherwise, at logical admission (section 4) take the held ops of every document the commit acquired and put them in front of its own -storage makes it durable and calls adopt(seq), inside Storage.commit() (section 4) - adopt every prepared change by pointer swap; record its telemetry events + join the writer's pending batch; release the line +the writer's storage call makes it durable (a receipt) and settles + after the commits before it: apply it to the read caches, adopt every prepared change by + pointer swap; record its telemetry events queue one microtask that calls observeCommits() observers with what was stored the Session's own observers, including watches and view mounts (section 9.3), capture it synchronously; the waits they resolve and the aborts they queue run after that microtask -storage resolves - fail the Session if an own observer threw (the commit stands); release the line -storage rejects after adopt(seq) - the commit stays adopted and published; the storage error fails the Session (section 10.1), + fail the Session if an own observer threw (the commit stands, the rest of the call is not published) +the call rejects after it stored the commit + the commit is published; the storage error rejects it and fails the Session (section 10.1), and an own observer's throw is reported -storage fails before adopt(seq) +the commit is not stored (refused, rejected, or after a failure) abort every prepared change, fail the Session (section 10.1), and publish nothing ``` @@ -1834,65 +1840,92 @@ granule, a tenth of the retention up to one second. Task documents also retire with their task (section 5.5). `createSession(storage, { documentRetentionMs, setTimer })` sets them for a plain Session; the Harness reads its settings at each use. -A Session commit callback may be asynchronous. It owns the Session mutation -line through callback execution, preparation, storage settlement, committed -baseline adoption, and publication. Adoption runs inside -`Storage.commit()`: the Session passes an `adopt(seq)` callback, which the -storage calls synchronously once the batch is durable and before its next -commit (section 10), so memory always shows a complete prefix of the log, plus -what deferred commits hold (section 4.1). External model, process, tool, -network, and human effects run outside the line. +A Session commit callback may be asynchronous. The Session mutation line runs +one job at a time: a commit's callback and its settlement, or a read on the +line. A commit with writes then joins the pending batch (logical admission), +and the line goes on with the next job. Storage overlaps the line (group +commit): a writer off the line sends every commit admitted while no storage +call is in flight as one `Storage.commit()` call (section 10), each with +its own `seq`, and publishes them one by one, in line order. Memory +always shows a complete prefix of the log, plus what deferred commits hold +(section 4.1). External model, process, tool, network, and human effects run +outside the line. -The commit is published when it is adopted, so every view changes in the same -step as memory. Inside `adopt`, after the trackers, come the commit's telemetry +No job observes what an unpublished commit wrote. Every read or write of a job +touches a key: a record (task, conversation, entry, submission, a +conversation-scoped request ID), a document address or incarnation, a scope's +documents, a conversation's newest entries, or, in the scheduler, the live work +below an ownership node, the ownerless roots, and the cancellation intent set; +a scan that cannot be keyed touches every key. While an admitted commit that +writes the key is unpublished, the job waits until it is stored and published, +then goes on; a job waiting holds the line, so the unpublished writes only +shrink. Code before a job's first such touch runs early: a callback can run +while an earlier commit is being stored, and cancellation is checked when the +job starts. The scheduler decides in regions that record the keys they read and +run again after such a wait (section 5.4). A commit's outcome is delivered once +it is published, and every job's outcome (commits with and without writes, +reads, deferred commits, callback errors) in line order; a scheduler commit +that writes nothing and waited at no gate delivers to the scheduler at once. A failure that fails +the Session makes every later outcome `SessionFailed`; a callback's own error, +a cancellation, or a refused read-only commit affects only its job. At most 256 +admitted jobs wait for their outcome; beyond that the line waits. The bound is a +count, not bytes: what waits is bounded by what one commit can hold (tool output +by its output limits), times the count. + +A commit is published after the storage call that stored it settled, so every +view changes in the same step as memory. A Storage with `publishing()` runs each +commit's publication inside a scope of its own: the store backend runs it in its +attached host's batch, so Chord sources over Durable's views are marked in one +batch per commit. Its publication applies it to the Session's read caches and +adopts it in memory; then come the commit's telemetry events (`pi.durable.entry`, `pi.durable.submission`), then the Session's own observers, synchronously: document watches, conversation view mounts, the task graph, the scheduler, and submission waits, with the observers adapters attach (`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, section 9.1). They advance watches and resolve waits, but run no task code: the abort signal of a task the commit newly marks follows in a microtask. Host `observeCommits()` observers run one microtask -after `adopt`, outside the storage's synchronous step, before every wait the -commit resolves and every abort it queues: those continuations are queued by -the own observers, after the dispatch. There is no order between host listeners -and watch listeners, nor `onWake`: they are independent observers of the -commit, and a watch delivery already queued for an earlier frame may deliver -this commit first. The host listeners run before the Session sees -`Storage.commit()` resolve, while the line is still held. A storage that works -after `adopt` before it resolves, as JSONL does when it reclaims sidecars, runs -what the commit wakes during that work too; its next commit still waits. +after the publication, before every wait the commit resolves and every abort it +queues: those continuations are queued by the own observers, after the +dispatch. There is no order between host listeners and watch listeners, nor +`onWake`: they are independent observers of the commit, and a watch delivery +already queued for an earlier frame may deliver this commit first. Two commits +of one storage call are published a macrotask apart, so the host listeners of +the first, and the continuations of the waits it resolves (`commit()`, +`Submission.wait()`, `waitForTask()`), run before the second is published. ```text -adopt(41): trackers, telemetry events, own observers (views advance, waits resolve) +commit([.., 41, 42]) settles +publish 41: caches, trackers, telemetry events, own observers (views advance, waits resolve) microtask: observeCommits() observers -microtasks after it: wait continuations, abort listeners +microtasks after it: wait continuations, session.commit() resolves, abort listeners microtasks, in no order relative to the above: watch listeners, onWake -Storage.commit() resolves; session.commit() resolves +macrotask: publish 42 ``` -`adopt` never throws. A throwing telemetry adapter is reported (`host_error`, +Publication never throws. A throwing telemetry adapter is reported (`host_error`, callback `telemetry`; a plain Session drops it, as it drops every report) and the commit resolves; a host listener's throw is reported (section 10.1), and an adapter's observer that throws is also detached and ended (section 9.1). The first throw of an own observer is kept, the other observers and the host -listeners still get the commit, and it fails the Session once -`Storage.commit()` returned; the commit stands and resolves. A throw while -memory adopts the batch is different: nothing is published, the Session fails -once `Storage.commit()` returned, and the commit rejects with that error. A -commit that Storage rejects after its adoption is durable: it stays adopted and -published, waits it resolved stay resolved, the commit rejects with the storage -error, and the Session fails with it; an own observer's throw in that commit is -then reported as a `listener_error`. +listeners still get the commit, and it fails the Session after the host +listeners got it; the commit stands and resolves, and the commits after it in +the same call are not published (section 10.1). A throw while memory adopts +the commit is different: nothing more of it is published, the Session fails, +and the commit rejects with that error. A commit Storage stored whose call then +rejects is durable: it is published, waits it resolved stay resolved, the +commit rejects with the storage error, and the Session fails with it; an own +observer's throw in that commit is then reported as a `listener_error`. `observeCommits(observer)` observes stored commits in one synchronous `observer.advance(publication, context)` call per observer, on the line, each with exactly what that storage transaction stored (section 4.1). It attaches synchronously and returns a `CommitObservation` with an idempotent `detach()`; unlike the other `observeXXX()` it has no `value`: commits are events, not a state, and the Session does -not know Storage's newest `seq` before its own first commit. Adoption +not know Storage's newest `seq` before its own first commit. A publication queues the one dispatch of a commit only when an observer is attached; the dispatch calls the observers attached when it runs. So an observer attached -between `adopt` and the dispatch gets the commit only when some observer was -attached at `adopt`; one detached in between does not get it; one attached after -the dispatch ran misses it. `observer.end(end)` comes once with a `WatchEnd`: `ended` when the Session has closed, +between the publication and the dispatch gets the commit only when some +observer was attached at the publication; one detached in between does not get +it; one attached after the dispatch ran misses it. `observer.end(end)` comes once with a `WatchEnd`: `ended` when the Session has closed, after its last publication, or `failed` with the error that failed it; never after `detach()`. An observer whose `advance` throws, or returns a promise that rejects, is reported (section 10.1), detached, and ended `listener_error`; the others still get the commit. `subscribeClose()` observes close synchronously when it @@ -1916,9 +1949,15 @@ await session.commit(async tx => { ``` Mutation admission occurs on the Session line before a commit callback starts. Closing seals mutation admission and task -reservation. Already-admitted commits settle before storage closes. Once a -commit is admitted, caller cancellation does not interrupt storage settlement or -undo the commit. Cancelling a close wait does not reopen admission. +reservation, runs the close listeners, and signals every task invocation; the +callbacks already admitted finish, waits at their touches included, every +pending and in-flight storage call settles and is published, the writes at +close and the held changes go through the same writer, and then storage closes. +Once a commit is admitted, caller cancellation does not interrupt storage +settlement or undo the commit. Cancelling a close wait does not reopen admission. +`unloadDocuments()`, a reload for another definition version, and the store at a +held document's deadline wait until every admitted commit is published, then +store what is held at that point through the writer. Table rules: @@ -1991,9 +2030,12 @@ What is held is stored by the first of: another definition version, which reloads the document from Storage; retirement is case 1. -Held operations leave their queue only at the last step before -`Storage.commit()`, after every fallible step: a callback, settlement, or -checkpoint failure of a normal commit keeps them. A store fails like any +Held operations leave their queue only at the commit's logical admission +(section 4), after every fallible step: a callback, settlement, or checkpoint +failure of a normal commit keeps them, and a commit never stores a hold adopted +after it in line order. A held commit is adopted in memory and published when +the commits before it are published, and what it writes stays unpublished until +then: a commit that reads the document waits for it. A store fails like any storage write: it fails the Session (section 10.1). Past Chord's `MAX_PENDING_OPS` (10,000) held operations, or when a held commit's `checkpointWhen` selected a base, the store writes a base of the current value. @@ -2002,7 +2044,7 @@ A failed Session stores nothing; after a reopen nothing is held. Publication: the Session's own observers see each held commit once, when memory adopts it, and of a normal commit that stores held changes only its own operations. `observeCommits()` sees every storage transaction with exactly -what it stored, published when Storage adopts it (section 4): nothing for a +what it stored, published once stored (section 4): nothing for a held commit, and the stored operations, held first, for a commit or a deadline that stores them. A document the commit retires is published as its retirement only: the held operations and the commit's own change stored before the @@ -2385,7 +2427,16 @@ or its `orphaned` settlement when its definition is unavailable. ### 5.4 Scheduler, abort, and ownership The scheduler serially reserves eligible tasks, then runs handlers off the -Session line. One in-memory `TaskInvocation` contains mode, abort controller, +Session line. Its decisions stay serialized on the line while their storage +overlaps (section 4): each reads the scheduler's memory, which changes as +commits are published, in a region that records what it reads (a task's +record, the live work below an ownership node, the ownerless roots, the +cancellation intent set) and stages nothing; when a commit that writes one of +those is unpublished, the region waits for it and runs again, so `migrate`, +registry snapshots, reports, and cleanup run once, after a decision stands. +Every commit writes the memberships its task changes alter: a task created or +ended changes the live work of every node from its parent up to the nearest +live task or the root conversation, and the chain is loaded at its admission. One in-memory `TaskInvocation` contains mode, abort controller, and completion promise. Abort protocol: @@ -3288,6 +3339,7 @@ consuming commit may rerun them. Abort errors always propagate. | hook | composition | ordinary throw | |---|---|---| | `beforeRequest` | replacement chain | report, continue | +| `beforeProviderRequest` | replacement chain | report, continue | | `afterResponse` | all observers | report, continue | | `onYield` | first continuation wins | report, continue | | `beforeTool` | argument replacement chain; first block wins | block tool with error text | @@ -3329,6 +3381,12 @@ interface GenerationHooks { api: HookApi, context: Context, ): HookResult<{ readonly messages: readonly Message[] }>; + /** After pi-ai builds the provider request body, before it is sent; replaces the body for that request only. */ + beforeProviderRequest( + request: { readonly model: Model; readonly payload: unknown }, + api: HookApi, + context: Context, + ): HookResult<{ readonly payload: unknown }>; /** Every terminal provider message, before classification. */ afterResponse(message: AssistantMessage, api: HookApi, context: Context): void | Promise; /** A final answer; `continue` appends a user message and continues the run. */ @@ -3750,8 +3808,14 @@ declares them, built from the resolved tools. Loaded tools follow the other tools in load order, so a load appends declarations to the transcript and the cached prefix holds. `@earendil-works/pi-durable/tool-search` provides the coding agent's `tool_search` (`createToolSearchExtension()`): it ranks -`Agent.deferred` with BM25 over the tools' names, descriptions, and parameter -names and descriptions, and loads the matches. The `codemode` section does not +`Agent.deferred` with the coding agent's tool ranking and loads the matches. +The ranking is BM25 over the tools' descriptions and parameter names and +descriptions, with the name as a separate field: a tool matching more of the +query's terms ranks above one matching fewer, a query term in the name +outweighs any number of mentions in the description, a tool whose whole name +after its last `__` is in the query gets a large bonus, and a query that is +exactly a tool's name ranks it first. Code mode's `searchTools()` ranks the +same way, with each tool's namespace text as a further, low-weight field. The `codemode` section does not list deferred tools; scripts find them with `searchTools()`. Tools reach files and processes only through `api.env`, never through an @@ -4638,9 +4702,10 @@ each commit writes at most one window, and the throttle bounds the rate. Generation creates `tools` with one `pending` slot per call in the commit that appends the tool-calling answer; a call its request did not offer is flagged -`unoffered`. A tool task sets its slot `running` in its intent commit, -publishes throttled output and details into it with deferred commits (section -4.1), and in its terminal commit sets +`unoffered`. A tool task sets its slot `running` (and a nested slot's final +arguments) with a deferred commit (section 4.1) right before its intent commit, +publishes throttled output and details into it with deferred commits, and in its +terminal commit sets `done` and `entry` and removes `output`, `droppedBytes`, `droppedLines`, `details`, and `diagnostics`, which the result entry now carries (section 7.3). The generation's `tools` phase removes `tools`. Slot updates apply only while a slot with the task's @@ -4823,7 +4888,9 @@ The run's inputs live in `pi.live.run`, not in the task input. `prepare`. `request` converts a leftover partial (below) before it resolves the model. - `request` first converts a committed partial left in `pi.live` by an - interrupted attempt into an aborted `pi.assistant` entry. When the host died + interrupted attempt into an aborted `pi.assistant` entry, and sets + `pi.live.generation` to the attempt with a deferred commit (section 4.1), which + the conversion makes immediate. When the host died while this request was in flight (the record's `restarts`, section 5.4), the attempt failed: that commit also either sleeps at `retry` until `now + retryDelayMs(policy, attempt)`, with `pi.live.generation.retry` saying @@ -4846,7 +4913,9 @@ The run's inputs live in `pi.live.run`, not in the task input. (section 4.1), so Storage gets them at the next commit that reads `pi.live` or within `settings.progress.persistWithinMs`. Before streaming, the `beforeRequest` chain may - replace the messages for this request only. After a close, recovery resends + replace the messages for this request only. Once pi-ai has built the provider + request body, the `beforeProviderRequest` chain may replace it, again for this + request only; it runs each time pi-ai builds a body. After a close, recovery resends the same committed messages with the same pinned model, thinking level, and stream options, and reruns `beforeRequest`; after a crash the attempt is charged instead, as above. @@ -4993,8 +5062,10 @@ Below, `{ entryId }` is a model-issued call's result. phase's agent `tools` (section 7.3), validates the arguments with pi-ai `validateToolArguments()`, runs the `beforeTool` chain, and validates again - (section 7.3). One commit then records intent: it moves to `execute` with the - final arguments and the tool's replay policy and sets the slot `running`. The + (section 7.3). A deferred commit sets the slot `running`; one commit then + records intent: it moves to `execute` with the final arguments and the tool's + replay policy. After a crash the slot may still show `pending` while the task's + checkpoint is `execute`; a safe rerun sets it `running` again. The same handler executes the tool, runs the `afterTool` chain, and commits the result; it builds `api.env` with `HarnessOptions.env` right before executing (section 2.2). A tool that does not resolve, invalid arguments, or a block @@ -5422,18 +5493,20 @@ interface Harness { Attaching works like acquiring a watch, atomically on the Session line: the observation's `value` includes every commit before it, and `advance` gets every later change that a watch would get, unfolded, as memory adopts it: for a -stored commit inside Storage's `adopt` (section 4), for a deferred one when +stored commit at its publication (section 4), for a deferred one when memory adopts it, before it is stored (section 4.1). A commit gives a view or the graph at most one call, and a document observer one call per change at its address: a commit that retires the document and creates it again gives two, the retirement and then the new incarnation's `[["r", value]]`. The task graph also advances without a commit, when a registry change or a failed migration changes a task's `blocked` reason (section 9.4): that call runs on the Session line, -outside any `adopt`, with a context of its own. A document observer follows its +outside any commit's publication, with a context of its own. A document observer follows its address as a document watch does, and `observeDoc()` gives `undefined` for an -absent one. `end` comes once: when the Session closes (`ended`) or fails -(`failed` with the error), or, for a task document, right after the retirement change of -the commit that makes its task terminal, inside that commit's `adopt` (`ended`). +absent one. `end` comes once: when the Session closes (`failed` with the error +"session closed": an adapter following the Session, such as Chord's `self.follow`, +fails and follows the next Session; watches end `ended`) or fails (`failed` with +the error), or, for a task document, right after the retirement change of the +commit that makes its task terminal, at that commit's publication (`ended`). `detach()` is idempotent, and nothing follows it, not even the rest of the commit underway; a detached observer is referenced by nothing of the Session. `context` governs only the attachment: cancelling it while attaching rejects @@ -5615,7 +5688,7 @@ The Harness keeps at most one mount per conversation. The first `watch()` or observer builds its documents on the Session line, and each attachment reads its window there with bounded scans; the mount is dropped when its last observer detaches. A mount derives each next revision from the Session's -commits as memory adopts them, for a stored commit inside Storage's `adopt` +commits as memory adopts them, for a stored commit at its publication (section 4) and for a held one when memory adopts it (section 4.1), like `watchDoc()`: it shows committed state, including deferred commits before they are stored (section 4.1). One publication that touches the conversation yields @@ -5785,11 +5858,11 @@ The Harness keeps at most one graph mount. The first `watchTaskGraph()` builds i live tasks and the conversations they own, one owner scan per live task, and holds the line while it does; the mount is dropped when its last observer detaches. It advances from the Session's stored commits as memory adopts them, -inside Storage's `adopt` (section 4), like the conversation view: a commit that +at their publication (section 4), like the conversation view: a commit that touches a conversation view and the graph changes both in that one step. A `blocked` reason that changes without a commit (a registry change, a failed migration) is derived again in a job on the Session line and published there, -outside any `adopt` and any commit, with the background context. One +outside any commit and its publication, with the background context. One publication that changes a node yields one Chord batch that sets or deletes `tasks[id]`, or sets a node's `conversations`. A publication that changes no node creates no revision. Task and conversation @@ -5911,12 +5984,27 @@ type StorageWrite = */ interface Storage { /** - * Persist the batch atomically. Once it is durable, call `adopt(seq)` exactly once, synchronously and before this - * storage's next commit starts, then resolve with `seq`; a commit that fails before the batch is durable never calls - * it. `adopt` adopts and publishes the batch: it runs the Session's own observers (section 4), which call no Storage - * method, and it never throws. + * Persist `batches` in order, each as one ordinary atomic commit with its own seq, checked against the state the + * earlier ones left, in as few round trips as the backend can. Every batch has at least one write: a commit without + * writes never reaches Storage, so the Session never sends an empty batch. Stops at the first batch Durable's own + * checks refuse (global IDs, document actions, copy resolution): the batches before it are durable, it and the ones + * after are not. Calls `receive(index, seq)` once per durable batch, in order, synchronously, after all of them are + * durable and before the next commit starts, and resolves with the number of durable batches and the refusal, if + * any. Rejects for an unknown outcome (connection lost, statement or commit failed); `receive` calls made before the + * rejection still stand. A receipt is not publication: the Session publishes later; it calls no Storage method and + * never throws. */ - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise<{ readonly stored: number; readonly refused?: unknown }>; + /** + * Optional. Run `publish`, the Session's publication of one commit (cache write-through, adoption, observers), + * inside this storage's own scope, such as a store host's batch. Call it exactly once, now or soon after; the + * Session publishes the next commit only once it ran. + */ + publishing?(publish: () => void, context: Context): void; /** * Allocate from the one global numeric namespace, above every ID minted since this storage was opened and every ID * stored, also across reopen: scans order by ID as creation order, and the Harness relies on it. The generic brand @@ -5947,6 +6035,19 @@ interface Storage { } ``` +Every backend implements `commit()` natively. It checks each batch (global +IDs, document actions, copy resolution) before that batch writes anything, so +a refusal leaves the earlier batches stored and nothing of its own; no +savepoints are needed, which Durable Objects' SQL does not allow. SQLite runs +one transaction per call. IndexedDB runs one `readwrite` transaction per call: +on a refusal it stops issuing writes and the transaction commits the prefix; +an error of IndexedDB itself aborts it, and the call rejects with nothing +stored. The store backend stores a call as one series of the store's commit +queue (`ChordStore.commitSeries`): one backend transaction, one seq and one +outbox commit per batch, the store's metadata written once. JSONL appends every batch's sidecar records and then +all their markers in one write (a torn append recovers a shorter complete +prefix), and memory applies them one by one. + Every error a method throws is final for the Session (section 10.1), so a backend retries its own transient failures. A read throws `StorageRequestError` for an invalid request, with no durable effect; from `commit()` it is fatal like any @@ -6058,12 +6159,35 @@ exempt: once a batch is admitted, whether it committed is unknown. A commit callback that catches a failed read cannot commit afterwards: the Session checks its health after the callback. Adopting a committed batch in memory that fails also fails the Session, since memory would be behind storage; so does a -`Storage.commit()` that resolves without adopting, or rejects after it adopted. -A batch Storage adopted is durable, so it is not rolled back: it was published -at adoption (section 4), `observeCommits()` observers get it and the waits it -settles resolve; then the commit rejects with the storage error and the Session -fails as above, with the storage error. An own observer that threw on that -commit is reported as a `listener_error`. +storage call that resolves without a receipt for a batch it reports stored. + +Each commit of one storage call (group commit, section 4) gets its own outcome, +in line order: + +- A commit Storage stored (a receipt) is published, then resolves. +- A call that stops at a refusal of commit `k`: the commits before `k` are + published and resolve; `k` rejects with the refusal; the commits after `k`, + and every commit admitted later, reject with `SessionFailed`, unpublished; + the Session fails with the refusal once the commits before `k` are + published. +- A call that rejects: the commits it stored are durable, so they are not + rolled back: they are published, `observeCommits()` observers get them and + the waits they settle resolve, and then they reject with the storage error, + as do the commits it did not store; the Session fails with that error. An own + observer that threw on such a commit is reported as a `listener_error`. +- A commit whose adoption or own observer throws: it rejects with the adoption + error, or stands and resolves for an observer's throw, and the Session fails; + the commits after it in the call are durable but not published, and reject + with `SessionFailed`. Reopening recovers them. +- A call that settles after another error already failed the Session, or a + commit of a call whose publication comes after another error failed it: what + it stored is durable but not published, and rejects with `SessionFailed` + whose cause is the first error, whatever the call returned. Reopening + recovers it; an `observeCommits()` consumer, such as a replica, misses it + until then. + +A job whose outcome comes after the one a failure is gets `SessionFailed`, even +when its callback ran before the failure; a job before it keeps its own outcome. **The Session's own bookkeeping fails it too.** A throw in a commit the scheduler makes for itself (reconciliation, reservation, a fault or terminal @@ -6073,9 +6197,9 @@ storage failure, a host callback such as a `RegistryReader` whose `snapshot()` throws, or a bug, and running the commit again fixes none of them. A throw in a commit listener of the Session's own components (scheduler, submissions, views, task graph, document observers) fails it as well, since their memory would fall -behind storage; the commit itself stands. Those listeners run inside Storage's -`adopt` (section 4), so the Session fails once `Storage.commit()` returned, -after the other observers and the `observeCommits()` observers got the commit. +behind storage; the commit itself stands. Those listeners run at the commit's +publication (section 4), so the Session fails after the other observers and the +`observeCommits()` observers got the commit. **User code fails only its unit.** A task phase that throws ends that task `faulted`; a throwing `migrate` blocks it as `migration_failed`; a tool that @@ -6142,7 +6266,8 @@ after a committed base or retirement. ### 11.2 SQLite -One SQL transaction is one Session commit. SQLite stores: +One SQL transaction is one storage call: the batches of a `commit()` call, +each checked before it writes. SQLite stores: - conversation, entry, task, and submission records; - document records; @@ -6180,7 +6305,9 @@ Publication protocol: 2. Append one complete main marker listing those records. 3. Publish in memory only after the marker write succeeds. -Every commit uses this protocol; there is no standalone-sidecar fast path. +Every commit uses this protocol; there is no standalone-sidecar fast path. A +`commit()` call appends every batch's sidecar records first, then all its +markers in one write of `main.jsonl`; reads wait while it appends. JSONL creation accepts an `fsync` option that defaults to `false`. Without `fsync`, it guarantees ordinary process-crash consistency, not survival of power, host, kernel, or filesystem failure. With `fsync: true`, the backend diff --git a/packages/durable/package.json b/packages/durable/package.json index 933fe6db7..c32091c32 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-durable", - "version": "1.4.0", + "version": "1.5.0", "description": "Durable conversation, task, and document runtime for Pi", "type": "module", "main": "./dist/index.js", @@ -123,7 +123,7 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run && PI_DURABLE_DOCUMENT_RETENTION_MS=0 vitest --run && npm run test:store && npm run test:workers", "test:store": "vitest --run --config vitest.store.config.ts", @@ -158,18 +158,17 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-codemode": "1.4.0", - "@earendil-works/pi-telemetry": "1.4.0", + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-codemode": "1.5.0", + "@earendil-works/pi-schema": "1.5.0", + "@earendil-works/pi-telemetry": "1.5.0", "@silvia-odwyer/photon": "0.3.3", - "diff": "8.0.4", - "typebox": "1.3.27" + "diff": "8.0.4" }, "devDependencies": { - "@cloudflare/vitest-plugin": "1.4.4", + "@cloudflare/vitest-plugin": "1.4.10", "fake-indexeddb": "6.2.6", - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/durable/src/codemode/codemode.ts b/packages/durable/src/codemode/codemode.ts index 2b308e2da..a2e765779 100644 --- a/packages/durable/src/codemode/codemode.ts +++ b/packages/durable/src/codemode/codemode.ts @@ -46,7 +46,7 @@ import type { CodemodeSandboxBase, } from "@earendil-works/pi-codemode/portable"; import { CODEMODE_SOURCE_GRAMMAR, parseCodemodeSource } from "@earendil-works/pi-codemode/source"; -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import { defineDoc } from "../documents.ts"; import type { ExecutionEnv } from "../env/index.ts"; import { errorMessage } from "../errors.ts"; @@ -210,7 +210,7 @@ export type CodemodeGlobal = { execute(args: unknown, api: ToolExecutionApi, context: Context): Promise; }; -const codemodeSchema = Type.Object({ code: Type.String({ description: "Raw JavaScript source." }) }); +const codemodeSchema = obj({ code: str({ description: "Raw JavaScript source." }) }, { open: true }); export type CodemodeToolInput = Static; @@ -946,14 +946,10 @@ class CodemodeCall { const documents = tools.flatMap((tool) => { const own = namespaceOfTool(tool.name); if (typeof namespace === "string" && (own === undefined || !matchesNamespace(own, namespace))) return []; - const text = [ - tool.name, - tool.name.replaceAll("_", " "), - tool.description, - own?.name ?? "", - own?.description ?? "", - ]; - return [{ name: tool.name, text: text.join(" ") }]; + const namespaceText = + own === undefined ? "" : [own.name, own.description ?? "", own.instructions ?? ""].join(" "); + // `description` is the tool's sample: its description and declaration, with the parameters' names and descriptions. + return [{ name: tool.name, text: tool.description, namespaceText }]; }); return rankDocuments(query, documents, limit).map((name) => { const tool = tools.find((candidate) => candidate.name === name)!; diff --git a/packages/durable/src/codemode/models.ts b/packages/durable/src/codemode/models.ts index ca22e96ea..fcdf63b68 100644 --- a/packages/durable/src/codemode/models.ts +++ b/packages/durable/src/codemode/models.ts @@ -17,7 +17,7 @@ import type { ModelTypeMap, Usage, } from "@earendil-works/pi-ai"; -import { Type } from "typebox"; +import { obj, unknown } from "@earendil-works/pi-schema"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -227,7 +227,7 @@ function resolveModel( ); } -const ModelCallParameters = Type.Object({ model: Type.Unknown(), context: Type.Unknown() }); +const ModelCallParameters = obj({ model: unknown(), context: unknown() }, { open: true }); /** The result of a model call as the tool returns it: the whole result for the script, usage counted by the Harness. */ function modelResult( @@ -255,7 +255,7 @@ export function createModelTools(): ToolRegistration[] { name: CLASSIFY_TOOL, description: "models.classify() of code mode scripts.", parameters: ModelCallParameters, - structuredOutputSchema: Type.Unknown(), + structuredOutputSchema: unknown(), callers: ["tools"], execute: async (args, api, context) => { const model = resolveModel("models.classify", "classifier", args.model, api.models); @@ -271,7 +271,7 @@ export function createModelTools(): ToolRegistration[] { name: GENERATE_IMAGES_TOOL, description: "models.generateImages() of code mode scripts.", parameters: ModelCallParameters, - structuredOutputSchema: Type.Unknown(), + structuredOutputSchema: unknown(), callers: ["tools"], execute: async (args, api, context) => { const model = resolveModel("models.generateImages", "image", args.model, api.models); diff --git a/packages/durable/src/harness/context.ts b/packages/durable/src/harness/context.ts index aa5660801..d41e1a176 100644 --- a/packages/durable/src/harness/context.ts +++ b/packages/durable/src/harness/context.ts @@ -17,27 +17,31 @@ export type ContextBounds = { }; /** - * Capture the bounds of the current context, or of the context cut off at the visible entry `at`, with two O(1) - * reads. Run this on the Session line; entries at or below the tail are immutable, so `deriveContext()` can then scan - * them off the line. + * Capture the bounds of the current context, or of the context cut off at the visible entry `at`. The newest entry + * and head marker are known from the Session's commits, so the current bounds usually cost no Storage read. Run this + * on the Session line; entries at or below the tail are immutable, so `deriveContext()` can then scan them off the + * line. */ export async function captureContextBounds( + session: SessionImpl, storage: Storage, conversationId: ConversationId, context: Context, at?: EntryId, + /** Set when `at` is known to be visible, as an entry of a range of the conversation is. */ + visible = false, ): Promise { let tail: EntryId | undefined; if (at === undefined) { - tail = (await storage.scanEntries({ conversationId }, 1, undefined, context)).items[0]?.id; + tail = await session.latestEntryId(conversationId, context); if (tail === undefined) return undefined; } else { - if ((await storage.entry(conversationId, at, context)) === undefined) { + if (!visible && (await storage.entry(conversationId, at, context)) === undefined) { throw new Error(`Entry ${at} is not visible from conversation ${conversationId}`); } tail = at; } - return { head: await storage.findLatestHeadMarker(conversationId, tail, context), tail }; + return { head: await session.latestHeadMarker(conversationId, tail, context), tail }; } /** @@ -60,6 +64,11 @@ export type ContextRange = { readonly open: readonly Message[]; /** The baseline's system message; see `findBaseline()`. */ readonly baseline: Baseline; + /** + * Whether the range reached the conversation's newest entry when it was read, so the entries of later commits extend + * it (`advanceRange()`); a range cut off at an earlier entry is not. + */ + readonly current: boolean; }; /** A baseline's system message, `null` when the range cannot have one, or `undefined` while a later entry may be one. */ @@ -68,7 +77,9 @@ type Baseline = Message | null | undefined; /** * `readContext()` that reuses `previous`, an earlier range of the same conversation: with the same head marker, only * entries after its tail are scanned. Returns the view and the range to pass to the next read. A range outlives the - * read, so its entries are frozen like `MemoryStorage` records, and the returned view has its own arrays. + * read, so its entries are frozen like `MemoryStorage` records, and the returned view has its own arrays. A caller that + * keeps the range for `advanceRange()` checks `current` again, synchronously where it installs it: a commit can be + * published between this function's return and the caller's continuation. */ export async function readContextFrom( session: SessionImpl, @@ -78,23 +89,33 @@ export async function readContextFrom( at: EntryId | undefined, previous: ContextRange | undefined, ): Promise<{ readonly view: ContextView; readonly range: ContextRange | undefined }> { - const bounds = await session.readOnLine(() => captureContextBounds(storage, conversationId, context, at)); + const visible = at !== undefined && previous !== undefined && rangeEntry(previous, at) !== undefined; + const bounds = await session.readOnLine((line) => + captureContextBounds(session, line, conversationId, context, at, visible), + ); if (bounds === undefined) return { view: emptyView(), range: undefined }; freezeJson(bounds.head); + const current = at === undefined; let range: ContextRange; if (previous === undefined || previous.bounds.head?.id !== bounds.head?.id) { - range = deriveRange(bounds, freezeJson(await scanRange(storage, conversationId, rangeQuery(bounds), context))); + const entries = freezeJson(await scanRange(storage, conversationId, rangeQuery(bounds), context)); + range = deriveRange(bounds, entries, current); } else if (bounds.tail === previous.bounds.tail) { range = previous; } else if (bounds.tail < previous.bounds.tail) { range = deriveRange( bounds, previous.entries.filter((entry) => entry.id <= bounds.tail), + false, ); } else { const minEntryId = idFromNumber(previous.bounds.tail + 1); const added = await scanRange(storage, conversationId, { minEntryId, maxEntryId: bounds.tail }, context); - range = extendRange(previous, bounds, freezeJson(added)); + range = extendRange(previous, bounds, freezeJson(added), current); + } + // Entries committed while the scan ran off the line are not in the range: it is current only if none was. + if (range.current && session.knownLatestEntryId(conversationId) !== range.bounds.tail) { + range = { ...range, current: false }; } const { view } = range; return { @@ -116,7 +137,7 @@ export async function readContext( context: Context, at?: EntryId, ): Promise { - const bounds = await session.readOnLine(() => captureContextBounds(storage, conversationId, context, at)); + const bounds = await session.readOnLine((line) => captureContextBounds(session, line, conversationId, context, at)); return deriveContext(storage, conversationId, bounds, context); } @@ -133,15 +154,49 @@ export async function deriveContext( context: Context, ): Promise { if (bounds === undefined) return emptyView(); - return deriveRange(bounds, await scanRange(storage, conversationId, rangeQuery(bounds), context)).view; + return deriveRange(bounds, await scanRange(storage, conversationId, rangeQuery(bounds), context), false).view; } function emptyView(): ContextView { return { head: undefined, entries: [], contributions: [], messages: [] }; } +/** Entry `id` of `range`, by binary search: a range holds its entries in ID order, every one visible from its conversation. */ +export function rangeEntry(range: ContextRange, id: EntryId): EntryRecord | undefined { + const entries = range.entries; + let low = 0; + let high = entries.length - 1; + while (low <= high) { + const middle = (low + high) >>> 1; + const entry = entries[middle]!; + if (entry.id === id) return entry; + if (entry.id < id) low = middle + 1; + else high = middle - 1; + } + return undefined; +} + +/** + * `range` with the entries one commit appended to its conversation, in ID order, without reading Storage: the + * conversation's own new entries are its newest, and a new head marker's head never lies before the range's start. + * A range that is not `current` stays as it is; the next read extends it from Storage. `undefined` drops the range. + */ +export function advanceRange(range: ContextRange, appended: readonly EntryRecord[]): ContextRange | undefined { + if (!range.current || appended.length === 0) return range; + const added = freezeJson(appended); + let head = range.bounds.head; + for (const entry of added) if (entry.head !== undefined) head = entry as typeof head; + const bounds = { head, tail: added.at(-1)!.id }; + if (head?.id === range.bounds.head?.id) return extendRange(range, bounds, added, true); + const start = head!.head; + // Heads only move forward: compaction keeps a later entry, a reset starts at itself, an older head is stale. + if (range.bounds.head !== undefined && start < range.bounds.head.head) return undefined; + const entries = [...range.entries, ...added].filter((entry) => entry.id >= start); + return deriveRange(bounds, entries, true); +} + /** Derive the context view from the entries scanned within `bounds`. */ -function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[]): ContextRange { +function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[], current: boolean): ContextRange { const edits = new Map(); // Edits of every entry in the range count, including older head markers that `selectActive()` drops. for (const entry of entries) for (const edit of entry.edits ?? []) edits.set(edit.target, edit); @@ -151,17 +206,22 @@ function deriveRange(bounds: ContextBounds, entries: readonly EntryRecord[]): Co const baseline = findBaseline(bounds.head, active); const messages = leadWithSystem([...settled, ...orderToolResults(open)], baseline); const view = { head: bounds.head, entries: active, contributions, messages }; - return { bounds, entries, view, edited: new Set(edits.keys()), settled, open, baseline }; + return { bounds, entries, view, edited: new Set(edits.keys()), settled, open, baseline, current }; } /** * `previous` extended by the entries `added` after its tail under the same head marker: only their contributions and * the open messages are derived. An added edit can change an earlier entry, so it derives the whole range again. */ -function extendRange(previous: ContextRange, bounds: ContextBounds, added: readonly EntryRecord[]): ContextRange { +function extendRange( + previous: ContextRange, + bounds: ContextBounds, + added: readonly EntryRecord[], + current: boolean, +): ContextRange { const entries = [...previous.entries, ...added]; if (added.some((entry) => entry.edits !== undefined || entry.head !== undefined || previous.edited.has(entry.id))) { - return deriveRange(bounds, entries); + return deriveRange(bounds, entries, current); } const contributions = added.map((entry) => contribute(entry, undefined)); const { settled, open } = settle(previous.settled, [...previous.open, ...contributions.flat()]); @@ -172,7 +232,7 @@ function extendRange(previous: ContextRange, bounds: ContextBounds, added: reado contributions: [...previous.view.contributions, ...contributions], messages: leadWithSystem([...settled, ...orderToolResults(open)], baseline), }; - return { bounds, entries, view, edited: previous.edited, settled, open, baseline }; + return { bounds, entries, view, edited: previous.edited, settled, open, baseline, current }; } /** diff --git a/packages/durable/src/harness/define.ts b/packages/durable/src/harness/define.ts index dd4bd2e8d..f5e0df8be 100644 --- a/packages/durable/src/harness/define.ts +++ b/packages/durable/src/harness/define.ts @@ -1,5 +1,4 @@ import type { JsonValue } from "@earendil-works/chord"; -import type { TSchema } from "@earendil-works/pi-ai"; import type { Task } from "../types.ts"; import type { AnyTask, @@ -20,7 +19,7 @@ export function defineExtension( +export function defineTool( tool: ToolRegistration, ): ToolRegistration { return tool; diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index b86af2ed0..17c28db47 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -242,7 +242,7 @@ export const GenerationTask = defineTask { // The context ends in a message the model answers, so the conversation has an entry. - let cutoff = (await tx.scanEntries({ conversationId }, 1)).items[0]!.id; + let cutoff = (await tx.latestEntryId(conversationId))!; for (const entry of entries) cutoff = (await tx.appendEntry(SystemEntry, conversationId, entry)).id; // Checked in this commit, so a compaction admitted during preparation counts. if (threshold === "background" && (await tx.doc(LiveDoc, conversationId)).compactions === undefined) { @@ -266,12 +266,18 @@ export const GenerationTask = defineTask { - const live = await tx.doc(LiveDoc, conversationId); - await convertPartial(tx, live, conversationId); - live.generation = { attempt }; - return undefined; - }, context); + // The attempt shown is presentation, stored like progress (`PROGRESS_COMMIT`); converting a partial left by an + // interrupted attempt appends an entry, which makes the commit immediate. + await runtime.commit( + async (tx) => { + const live = await tx.doc(LiveDoc, conversationId); + await convertPartial(tx, live, conversationId); + live.generation = { attempt }; + return undefined; + }, + context, + PROGRESS_COMMIT, + ); const model = runtime.models.getModel(ref.provider, ref.modelId); if (model === undefined) return failNoModel(runtime, ref, context); const view = await runtime.context(conversationId, context, { at: cutoff }); @@ -286,6 +292,7 @@ export const GenerationTask = defineTask providerPayload(runtime, model, payload, context), }; const message = await streamResponse(runtime, model, messages, options, attempt, context); const request = { attempt, compacted, model: ref, thinkingLevel, cutoff, messages: view.messages }; @@ -658,6 +665,21 @@ export async function convertPartial(tx: Tx, live: Draft, conversatio await appendAssistant(tx, conversationId, { ...message, stopReason: "aborted" }); } +/** The `beforeProviderRequest` replacement chain over one provider request body. */ +async function providerPayload( + runtime: Runtime, + model: Model, + payload: unknown, + context: Context, +): Promise { + let current = payload; + await runtime.hooks.each("beforeProviderRequest", async (hook) => { + const replaced = await hook({ model, payload: current }, runtime, context); + if (replaced !== undefined) current = replaced.payload; + }); + return current; +} + /** * Stream one request and return the terminal message. Partials commit as trailing writes at most every * `progress.partialIntervalMs` (default 100 ms) with one commit in flight, deferred (`PROGRESS_COMMIT`); `finally` stops diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 906fb3539..8fb91bcc0 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -106,6 +106,9 @@ type ConversationHost = { /** Key of the Harness's internal index-size getter, for tests. */ export const SCHEDULER_INDEX_SIZES = Symbol.for("pi-durable.schedulerIndexSizes"); +/** Key of the Harness's internal idle-state getter (kept contexts against a full recompute), for tests. */ +export const SCHEDULER_IDLE_STATE = Symbol.for("pi-durable.schedulerIdleState"); + class ConversationImpl implements Conversation { readonly id: ConversationId; readonly #host: ConversationHost; @@ -161,7 +164,7 @@ class ConversationImpl implements Conversation { context: Context, ): Promise> { const scan = conversationEntries(this.id, query); - return this.#host.harness.readOnLine(() => this.#host.storage.scanEntries(scan, limit, cursor, context)); + return this.#host.harness.readOnLine((storage) => storage.scanEntries(scan, limit, cursor, context)); } fork(at: EntryId, options: ConversationCreateOptions, context: Context): Promise { @@ -249,14 +252,14 @@ class HarnessImpl extends SessionImpl implements settleOutcome: settleSchedulerOutcome, withdrawInputs: withdrawQueuedInputs, conversation: async (id, binding, callContext) => { - const record = await this.readOnLine(() => storage.conversation(id, callContext)); + const record = await this.readOnLine((line) => line.conversation(id, callContext)); return record === undefined ? undefined : boundConversation(id, binding, this.#submissions, this.#tasks); }, context: withoutAbortSignal(context), readOnly: options.readOnly === true, }); - this.#submissions = new Submissions(this, storage, now, settings, () => this.#tasks.resume(), options.telemetry); - this.#taskGraph = new TaskGraphView(this, storage, { + this.#submissions = new Submissions(this, now, settings, () => this.#tasks.resume(), options.telemetry); + this.#taskGraph = new TaskGraphView(this, { blocked: (record) => this.#tasks.blockedReason(record), subscribeBlocked: (listener) => this.#tasks.subscribeBlocked(listener), }); @@ -265,7 +268,7 @@ class HarnessImpl extends SessionImpl implements storage, tasks: this.#tasks, submissions: this.#submissions, - views: new ConversationViews(this, storage), + views: new ConversationViews(this), now, create: (target, createOptions, context) => this.#create(target, createOptions, context), }; @@ -308,7 +311,7 @@ class HarnessImpl extends SessionImpl implements } getTask(id: TaskId, context: Context): Promise | undefined> { - return this.readOnLine(() => this.storage.task(id, context)) as Promise< + return this.readOnLine((storage) => storage.task(id, context)) as Promise< TaskRecord | undefined >; } @@ -318,11 +321,16 @@ class HarnessImpl extends SessionImpl implements return this.#tasks.indexSizes; } + /** Internal, for tests: the kept contexts' idle state and its disagreements with a full recompute. */ + get [SCHEDULER_IDLE_STATE](): TaskScheduler["idleState"] { + return this.#tasks.idleState; + } + inspect(context: Context): Promise { - return this.readOnLine(async () => { + return this.readOnLine(async (storage) => { const { scheduling, tasks } = await this.#tasks.inspect(this.#options.registry.snapshot()); const scan = (status: "queued" | "placed") => - scanAll((cursor) => this.storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context)); + scanAll((cursor) => storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context)); const submissions = [...(await scan("queued")), ...(await scan("placed"))].sort((a, b) => a.id - b.id); return { scheduling, tasks, submissions }; }); @@ -356,8 +364,8 @@ class HarnessImpl extends SessionImpl implements /** Sum every conversation's committed `pi.usage`. Each document is read at its own point; totals only grow. */ async usage(context: Context): Promise { - const conversations = await this.readOnLine(() => - scanAll((cursor) => this.storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)), + const conversations = await this.readOnLine((storage) => + scanAll((cursor) => storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)), ); const total = UsageDoc.definition.initial(); for (const { id } of conversations) { @@ -395,7 +403,7 @@ class HarnessImpl extends SessionImpl implements async conversation(id: ConversationId, context: Context): Promise { this.assertUsable(); - const record = await this.readOnLine(() => this.storage.conversation(id, context)); + const record = await this.readOnLine((storage) => storage.conversation(id, context)); return record === undefined ? undefined : new ConversationImpl(record.id, this.#host); } @@ -407,6 +415,11 @@ class HarnessImpl extends SessionImpl implements this.#report(error, info, context); } + /** The ownership memberships a commit's task changes write, from the scheduler's index. */ + protected override membershipKeys(tx: Transaction): readonly string[] | Promise { + return this.#tasks.membershipKeys(tx); + } + /** Signal and join task invocations after the close listeners and before Storage closes; writes no task outcome. */ protected override beforeClose(): Promise { return this.#tasks.join(); diff --git a/packages/durable/src/harness/registry-reader.ts b/packages/durable/src/harness/registry-reader.ts new file mode 100644 index 000000000..97f8a44bd --- /dev/null +++ b/packages/durable/src/harness/registry-reader.ts @@ -0,0 +1,34 @@ +// A Harness's registry over a Chord registry (Chord v2 slice 8 §5): the composed value of p3's extensions registry, +// read as Durable's `RegistryReader`. + +import type { Context } from "@earendil-works/chord/context"; +import type { RegistryValue } from "@earendil-works/chord/next"; +import type { RegistryReader, RegistrySnapshot, ToolRegistration } from "./types.ts"; + +/** + * A `RegistryReader` over a Chord registry value whose composition is a registry snapshot: `snapshot()` reads the + * current composition (the last good one while `compose` fails), `subscribe` observes its compositions, its listener + * called synchronously inside the Chord batch that composed (so it only wakes the scheduler; a Chord call there + * throws `invalid/reentrant`). `context` is the observation's: aborting it detaches every listener. + */ +export function registryReader( + value: RegistryValue>, + context: Context, +): RegistryReader { + return { + snapshot: () => value.value, + subscribe: (listener) => { + const observation = value.observe( + { + advance: (call) => { + // A failed composition keeps the last snapshot: nothing to reconsider. + if (call.kind === "value") listener(); + }, + end: () => {}, + }, + context, + ); + return () => observation.detach(); + }, + }; +} diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index fc3c2107c..63cbe89ac 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -11,12 +11,14 @@ import { } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv } from "../env/index.ts"; import { errorMessage, ReadOnlySession } from "../errors.ts"; +import { ALL_KEYS, conversationKey, INTENT_KEY, ownedKey, ROOTS_KEY, taskKey } from "../session/keys.ts"; import type { ObservedCommit, SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import type { Timers } from "../timers.ts"; import type { CommitOptions, ConversationId, + ConversationRecord, DocumentWatch, EntryId, EntryRecord, @@ -37,7 +39,7 @@ import type { TaskState, } from "../types.ts"; import { agentHooks, DEFAULT_RESTART_LIMIT } from "./agent.ts"; -import { type ContextRange, readContextFrom } from "./context.ts"; +import { advanceRange, type ContextRange, rangeEntry, readContextFrom } from "./context.ts"; import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.ts"; import type { Agent, @@ -66,6 +68,7 @@ type ErasedRuntime = TaskRuntime; type ErasedRunningTask = RunningTask; const SCAN_PAGE_SIZE = 256; +const NO_KEYS: readonly string[] = []; const LIVE_STATUSES = ["pending", "running", "waiting", "sleeping", "completing"] as const; /** What `blockedReason()` reads of a task: the task graph keeps only this of each record. */ @@ -140,8 +143,22 @@ type ReportedTask = { readonly task: AnyTask | undefined } | undefined; /** Outcome of the phase that just returned, judged by the next step. */ type PhaseResult = { readonly checkpoint: Checkpoint; readonly failure?: { readonly error: unknown } }; -/** Step decision: continue with the next phase, end the invocation, or end it by writing `faulted`. */ -type Decision = boolean | { readonly fault: unknown }; +/** + * Step decision, made in a decision region: continue with the next phase, end the invocation, or end it by writing + * `faulted`. + */ +type StepPlan = + | { readonly kind: "continue" | "progress" | "end" | "cascade" } + | { readonly kind: "fault"; readonly error: unknown }; + +/** Continue with the next phase, the first one. */ +const CONTINUE: StepPlan = { kind: "continue" }; +/** Rule 5: the phase made durable progress; `#progress()` decides once the decision stands. */ +const PROGRESS: StepPlan = { kind: "progress" }; +/** End the invocation without a write. */ +const END: StepPlan = { kind: "end" }; +/** End it: an owner's cancellation intent reaches it before its cascade did; schedule the cascade. */ +const CASCADE: StepPlan = { kind: "cascade" }; /** Terminal outcomes the scheduler writes without running task code. */ export type SchedulerOutcome = Extract, { readonly status: "faulted" | "orphaned" }>; @@ -169,6 +186,9 @@ type Overlay = { links?: ReadonlyMap>; }; +/** Tasks a finalize pass has made terminal in its commit so far. */ +type Finalized = { has(id: TaskId): boolean }; + /** A node of the downward index: a task, or a conversation as `c`. */ type Node = TaskId | `c${number}`; @@ -251,9 +271,16 @@ export type TaskSchedulerOptions = { * later below a dropped node finds it unknown and loads its chain from storage again. * * Invariant: every task transition is decided and written by one callback serialized on the Session line. That covers - * reservation, marks, runtime commits, finalization, and the synchronous step before each phase, which applies the - * precedence rules and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the - * step that decides its end, so a runtime commit it queued either lands before that decision or is rejected. + * reservation, marks, runtime commits, finalization, and the step before each phase, which applies the precedence rules + * and writes a fault or handover. Handlers and joins run off the line. An invocation ends inside the step that decides + * its end, so a runtime commit it queued either lands before that decision or is rejected. + * + * Decisions stay serialized; their storage overlaps (group commit). The Session stores the commits of several jobs in + * one call while the line runs on, and this memory changes only as each commit is published. So every decision reads + * it in a decision region (`#region()`): synchronous, recording the keys it reads (a task's record, the live work below + * a node, the cancellation intent set), staging nothing; when one of them has an unpublished writer, the region waits + * for its publication and runs again, and only a region that read nothing unpublished is applied. Every commit writes + * the memberships its task changes alter (`membershipKeys()`), so a walk that a pending commit would change waits. */ export class TaskScheduler { readonly #telemetry: NonNullable; @@ -283,7 +310,24 @@ export class TaskScheduler { * newer entries. `idleSince` is the Harness time it was first seen idle. Derived and never persisted; dropped at the * first idle check after `settings.contextRetentionMs` of idleness, and at close. */ - readonly #contexts = new Map(); + readonly #contexts = new Map< + ConversationId, + { range: ContextRange; idleSince: number | undefined; expiresAt?: number } + >(); + /** + * The idle kept contexts by `idleSince`, oldest first: the expiry order, so neither an expiry nor finding the next one + * scans `#contexts`. Kept sorted also when the Harness clock goes back (see `#markIdle`). + */ + readonly #idleOrder = new Map(); + /** At least the newest `idleSince` in `#idleOrder` (drops do not lower it): an older one added means the clock went back. */ + #idleNewest = Number.NEGATIVE_INFINITY; + /** + * A chain load resolved what `#idle()` treats as unknown for a live task whose chain is not loaded, which can make a + * scope idle without a task change below it; the next idle check then covers every scope. + */ + #unknownsChanged = false; + /** Idle checks that covered every scope (`idleState`, a test diagnostic). */ + #fullIdleChecks = 0; /** Background timer for the earliest expiry of an idle context, unless the host declined it (see `#scheduleExpiry`). */ #expiry: { readonly at: number; readonly disarm: () => void } | undefined; readonly #invocations = new Map(); @@ -341,6 +385,13 @@ export class TaskScheduler { /** Reconcile passes underway with work (they may overlap): their commits finalize held outcomes and mark cascades. */ #reconcilingWork = 0; #sweepScheduled = false; + /** + * Nodes chain loads installed for jobs whose commits are not published yet, with how many such jobs: the sweep keeps + * them, as work a pending commit creates below them needs them. + */ + readonly #pins = new Map(); + /** Keys the decision region underway reads (`#region()`); set only while one runs. */ + #reading: Set | undefined; #cascadePending = false; #unsubscribeRegistry: () => void = () => {}; #enabled = false; @@ -387,6 +438,7 @@ export class TaskScheduler { this.#kick(); }); await this.#session.commitWith(async (tx) => { + await this.#published(); // Every table read before the first write. const scans = []; for (const status of LIVE_STATUSES) { @@ -465,8 +517,22 @@ export class TaskScheduler { const now = this.#now(); // A reconcile pass with work to do is work too: it finalizes held outcomes and marks cascades, at open as // recovery. A process that dies before it lands must be woken again. A pass with nothing to do is not. - let active = this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork(); + const active = + this.#invocations.size > 0 || this.#reconcilingWork > 0 || this.#reconcileWork() || this.#runsNow(now); + // The earliest sleep not yet due: only `#sleeping` holds sleeping tasks, so no pass visits every live task. let at: number | undefined; + for (const id of this.#sleeping) { + const record = this.#live.get(id); + const state = record?.state; + if (state?.status === "sleeping" && !record!.abortRequested && state.until > now) { + at = at === undefined ? state.until : Math.min(at, state.until); + } + } + return at === undefined ? { active } : { active, at }; + } + + /** Whether a live task can run now: the scan `wake()` needs only while no invocation runs. */ + #runsNow(now: number): boolean { let snapshot: RegistrySnapshot | undefined; let snapshotFailed = false; for (const record of this.#live.values()) { @@ -474,11 +540,7 @@ export class TaskScheduler { if (state.status === "completing") continue; if (state.status === "waiting" && !record.abortRequested && state.on.some((id) => this.#live.has(id))) continue; - if (state.status === "sleeping" && !record.abortRequested && state.until > now) { - at = at === undefined ? state.until : Math.min(at, state.until); - continue; - } - if (active) continue; + if (state.status === "sleeping" && !record.abortRequested && state.until > now) continue; // An abort invocation waits for the live work below it, which counts itself. if (record.abortRequested && this.#waitingOn(record, true).length > 0) continue; // A task no definition can take waits for a registry change, except an abort-marked one, which is orphaned. @@ -493,9 +555,9 @@ export class TaskScheduler { } if (snapshot !== undefined && "reason" in this.#fit(record, snapshot.task(record.kind))) continue; } - active = true; + return true; } - return at === undefined ? { active } : { active, at }; + return false; } /** @@ -538,12 +600,14 @@ export class TaskScheduler { if (keepRestart && current.abortReason === "restart") return { result: "marked" as const }; if (invocation === undefined && current.state.status !== "completing") { await this.#loadScopes(false); - if (!this.#hasOwnedLive(id)) { - const resolution = this.#resolve(current as RunnableTaskRecord, this.#registry.snapshot()); - if (resolution.kind === "blocked") { - await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason }); - return { result: "marked" as const }; - } + const owns = await this.#region(() => this.#hasOwnedLive(id), this.#reloadScopes); + // Resolved once that decision stands, as it may run `migrate`. + const resolution = owns + ? undefined + : this.#resolve(current as RunnableTaskRecord, this.#registry.snapshot()); + if (resolution?.kind === "blocked") { + await this.#terminate(tx, current, { status: "orphaned", reason: resolution.reason }); + return { result: "marked" as const }; } } if (!current.abortRequested || current.abortReason !== undefined) tx.setTask(withAbortMark(current)); @@ -559,10 +623,12 @@ export class TaskScheduler { async waitForTask(id: TaskId, context: Context): Promise> { // Check and register on the line so no terminal publication falls between them. - const found = await this.#session.readOnLine(async () => { + const found = await this.#session.readOnLine(async (storage) => { if (this.#closing) throw closedError(this.#session); - if (this.#live.has(id)) return { promise: this.#taskWaiters.add(id, context) }; - const record = await this.#storage.task(id, context); + if (await this.#region(() => this.#liveRecord(id) !== undefined)) { + return { promise: this.#taskWaiters.add(id, context) }; + } + const record = await storage.task(id, context); if (record === undefined) throw new Error(`Task ${id} does not exist`); return { promise: Promise.resolve(record as SettledTask) }; }); @@ -587,6 +653,7 @@ export class TaskScheduler { */ async abortConversation(conversationId: ConversationId, background: boolean, context: Context): Promise { const reached = await this.#session.commitWith(async (tx) => { + await this.#published(); const queued = await this.#loadScopes(true); const scope = { conversation: conversationId }; const reached: TaskId[] = []; @@ -605,14 +672,104 @@ export class TaskScheduler { await this.waitForIdle(conversationId, context); } + /** + * The ownership memberships a commit's task and conversation changes write (group commit §4.2), at its admission on + * the line: for each task created or made terminal, every node from its parent up to and including the nearest live + * task, or the root conversation, and then the roots too, conservatively, as a pending end may unlist the root a + * creation finds listed; for each conversation created under a task, that task; and the intent set when + * a change sets, upgrades, or clears a mark or a held or ended outcome other than `completed`. Loads the chains it + * walks; the nodes it installs stay until the commit is published. + */ + membershipKeys(tx: Transaction): readonly string[] | Promise { + const tasks = tx.stagedTasks(); + const conversations = tx.stagedConversations(); + if (tasks.length === 0 && conversations.every((record) => record.owner === undefined)) return NO_KEYS; + return this.#membershipKeys(tx, tasks, conversations); + } + + async #membershipKeys( + tx: Transaction, + tasks: readonly AnyTaskRecord[], + conversations: readonly ConversationRecord[], + ): Promise { + const keys = new Set(); + const overlay = overlayOf(tx); + const created = new Set(tx.createdTaskIds()); + const reached = new Map(); + for (const record of tasks) { + if (intentChanges(this.#live.get(record.id), record)) keys.add(INTENT_KEY); + if (!created.has(record.id) && record.state.status !== "terminal") continue; + await this.#loadChain(parentOf(record), overlay); + if ((await this.#chainKeys(parentNode(record), overlay, keys, reached)) !== undefined) keys.add(ROOTS_KEY); + } + for (const conversation of conversations) { + const owner = conversation.owner?.taskId; + if (owner === undefined) continue; + await this.#loadChain({ task: owner }, overlay); + await this.#chainKeys(owner, overlay, keys, reached); + } + return [...keys]; + } + + /** + * Add the memberships from `start` up to and including the nearest live task, or the root conversation, which it + * returns. `reached` keeps, per node walked, where its walk ended, so a chain several tasks of a commit share is + * walked once. Whether an ancestor is live can change under a pending commit, so it is read after a touch of the + * ancestor's record: a pending end must not stop the chain early. Ownership fields are immutable, so not keyed. + */ + async #chainKeys( + start: Node, + overlay: Overlay, + keys: Set, + reached: Map, + ): Promise { + const walked: string[] = []; + let root: ConversationId | undefined; + let node: Node | null | undefined = start; + while (node !== undefined && node !== null) { + const key = ownedKey(node); + // Walked up from here already, for another task of the commit. + if (reached.has(key)) { + root = reached.get(key); + break; + } + keys.add(key); + walked.push(key); + if (typeof node === "string") { + const conversation = conversationOf(node); + node = this.#edge(conversation, overlay); + if (node === null) root = conversation; + continue; + } + let record: AnyTaskRecord | undefined = overlay.tasks.get(node); + if (record === undefined) { + const gate = this.#session.touch(taskKey(node)); + if (gate !== undefined) { + await gate; + // It may have ended meanwhile, and its node been swept: load the chain above it again. + await this.#loadChain({ task: node }, overlay); + } + record = this.#live.get(node); + } + if (record !== undefined && record.state.status !== "terminal") break; + const above: TaskNode | undefined = record ?? this.#settled.get(node); + node = above === undefined ? undefined : parentNode(above); + } + for (const key of walked) reached.set(key, root); + return root; + } + // ─── Scheduling ──────────────────────────────────────────────────────── #observe(publication: ObservedCommit): void { // A commit held in memory changes documents only. if (publication.kind === "memory") return; + this.#advanceContexts(publication); const updated: AnyTaskRecord[] = []; const failed: TaskId[] = []; let changed = false; + // The conversations whose idleness the commit's task changes can change; undefined when a chain is not known. + let dirty: Set | undefined = new Set(); // Edges first, so the tasks a commit creates in a new conversation hang from it. for (const change of publication.changes) { if (change.type === "conversation" && !this.#edges.has(change.value.id)) { @@ -626,6 +783,10 @@ export class TaskScheduler { const previous = this.#live.get(record.id); if (failedOutcome(record) && (previous === undefined || !failedOutcome(previous))) failed.push(record.id); this.#track(previous, record); + // Idleness counts live tasks, so only a task that became live or ended can change it. Collected before `#sweep()`, + // which may forget the chain of a task that ended. + const liveness = previous === undefined || record.state.status === "terminal"; + if (liveness && dirty !== undefined && !this.#addScopesAbove(record, dirty)) dirty = undefined; if (record.state.status === "terminal") { this.#failedMigrations.delete(record.id); this.#failFastChecks.delete(record.id); @@ -637,7 +798,7 @@ export class TaskScheduler { if (record.abortRequested && previous?.abortRequested !== true) { this.#cascadePending = true; // Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which - // never runs inside Storage's `adopt`: the signal follows in a microtask, after host commit listeners. + // never runs inside a publication: the signal follows in a microtask, after host commit listeners. const invocation = this.#invocations.get(record.id); if (invocation?.mode === "run") queueMicrotask(() => invocation.controller.abort()); } @@ -686,10 +847,53 @@ export class TaskScheduler { if (this.#cascadePending) this.#scheduleReconcile(); this.#sweep(); if (!changed) return; - this.#settleIdle(); + this.#settleIdle(dirty); this.#kick(); } + /** + * Add the conversations above `record` to `scopes`: those whose idleness a change of it can change. False when a node + * on the way up is not known. A conversation already in `scopes` has everything above it there too. + */ + #addScopesAbove(record: AnyTaskRecord, scopes: Set): boolean { + for (const step of this.#above(parentOf(record))) { + if ("unknown" in step) return false; + if (!("conversation" in step)) continue; + if (scopes.has(step.conversation)) return true; + scopes.add(step.conversation); + } + return true; + } + + /** + * Extend each kept context range with the entries a stored commit appended to its conversation, so a later read + * needs no Storage scan. Entries are published in ID order. + */ + #advanceContexts(publication: Extract): void { + if (this.#contexts.size === 0) return; + let appended: Map | undefined; + for (const change of publication.changes) { + if (change.type !== "entry" || !this.#contexts.has(change.value.conversationId)) continue; + appended ??= new Map(); + const entries = appended.get(change.value.conversationId); + if (entries === undefined) appended.set(change.value.conversationId, [change.value]); + else entries.push(change.value); + } + if (appended === undefined) return; + for (const [conversationId, entries] of appended) { + const kept = this.#contexts.get(conversationId)!; + // An idle range past its retention is dropped here too: entry-only commits do not run `#settleIdle()`, and a + // host may decline the expiry timer. `expiresAt` was fixed when it went idle, so no settings are read here. + if (kept.idleSince !== undefined && kept.expiresAt !== undefined && kept.expiresAt <= this.#now()) { + this.#dropContext(conversationId); + continue; + } + const range = advanceRange(kept.range, entries); + if (range === undefined) this.#dropContext(conversationId); + else kept.range = range; + } + } + /** * `settings.contextRetentionMs`, or 0 when the host's settings throw. Kept contexts are only a cache, so dropping them * is safe, while a throw here would escape commit listeners, reconciliation, and the expiry timer. @@ -703,47 +907,98 @@ export class TaskScheduler { } } - /** Resolve idle waiters, and drop each kept context whose conversation has been idle for the retention period. */ - #settleIdle(): void { - for (const conversationId of this.#idleWaiters.keys()) { - if (this.#idle(conversationId)) this.#idleWaiters.resolve(conversationId); - } + /** + * After task changes below the conversations in `scopes` (every conversation when undefined): resolve their idle + * waiters and the Harness-wide ones, drop the kept contexts whose idle period passed the retention, then start or end + * the idle periods of the kept contexts in `scopes`. A task change can change the idleness only of the conversations + * above it, so no check visits every conversation. + */ + #settleIdle(scopes: ReadonlySet | undefined): void { + const all = scopes === undefined || this.#unknownsChanged; + this.#unknownsChanged = false; + if (all) this.#fullIdleChecks++; + this.#resolveIdleWaiters(all ? undefined : scopes); const now = this.#now(); const retention = this.#contextRetentionMs(); - for (const [conversationId, kept] of this.#contexts) { - if (kept.idleSince !== undefined && now - kept.idleSince >= retention) { - this.#contexts.delete(conversationId); - } else if (!this.#idle(conversationId)) { + // Expired first: a context past its retention is dropped even when the same commit makes it busy again. + this.#expireContexts(now, retention); + for (const conversationId of all ? [...this.#contexts.keys()] : scopes) { + const kept = this.#contexts.get(conversationId); + if (kept === undefined) continue; + if (!this.#idle(conversationId)) { + if (kept.idleSince === undefined) continue; kept.idleSince = undefined; - } else if (retention > 0) { - kept.idleSince ??= now; - } else { - this.#contexts.delete(conversationId); + this.#idleOrder.delete(conversationId); + } else if (retention <= 0) { + this.#dropContext(conversationId); + } else if (kept.idleSince === undefined) { + kept.idleSince = now; + kept.expiresAt = now + retention; + this.#markIdle(conversationId, now); } } this.#scheduleExpiry(); } + /** Resolve the idle waiters of the conversations in `scopes` (all when undefined) and the Harness-wide ones. */ + #resolveIdleWaiters(scopes: ReadonlySet | undefined): void { + for (const conversationId of this.#idleWaiters.keys()) { + if (conversationId !== undefined && scopes !== undefined && !scopes.has(conversationId)) continue; + if (this.#idle(conversationId)) this.#idleWaiters.resolve(conversationId); + } + } + /** - * Run `#settleIdle()` when the earliest idle context expires, on a background timer, which never keeps the process - * alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer could - * keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check expiry. + * Drop the kept contexts idle for at least `retention`, oldest first, with the retention setting of now; with no + * retention, every idle one, also those whose idle period a clock that went back makes start after now. + */ + #expireContexts(now: number, retention: number): void { + for (const [conversationId, idleSince] of this.#idleOrder) { + if (retention > 0 && now - idleSince < retention) return; + this.#dropContext(conversationId); + } + } + + /** + * Start the idle period of a kept context in the expiry order. An idle period older than the newest one kept, which + * a clock that went back starts, sorts the order again, and the newest is taken from what is kept then. + */ + #markIdle(conversationId: ConversationId, idleSince: number): void { + this.#idleOrder.delete(conversationId); + this.#idleOrder.set(conversationId, idleSince); + if (this.#idleOrder.size === 1 || idleSince >= this.#idleNewest) { + this.#idleNewest = idleSince; + return; + } + const sorted = [...this.#idleOrder].sort((a, b) => a[1] - b[1]); + this.#idleOrder.clear(); + for (const [id, since] of sorted) this.#idleOrder.set(id, since); + this.#idleNewest = sorted.at(-1)![1]; + } + + /** Forget a kept context, idle or not, and its place in the expiry order. */ + #dropContext(conversationId: ConversationId): void { + this.#contexts.delete(conversationId); + this.#idleOrder.delete(conversationId); + } + + /** + * Drop the expired contexts when the earliest idle context expires, on a background timer, which never keeps the + * process alive. Where the host declines it, as the default does in Cloudflare Workers, none is kept: a pending timer + * could keep a Durable Object from being evicted, and eviction frees the contexts. There, task changes alone check + * expiry. */ #scheduleExpiry(): void { - const retention = this.#contextRetentionMs(); - let at: number | undefined; - for (const kept of this.#contexts.values()) { - if (kept.idleSince !== undefined && (at === undefined || kept.idleSince + retention < at)) { - at = kept.idleSince + retention; - } - } + const oldest = this.#idleOrder.values().next(); + const at = oldest.done === true ? undefined : oldest.value + this.#contextRetentionMs(); if (this.#expiry !== undefined && this.#expiry.at === at) return; this.#expiry?.disarm(); this.#expiry = undefined; if (at === undefined || this.#closing) return; const disarm = this.#timers.background(() => { this.#expiry = undefined; - this.#settleIdle(); + this.#expireContexts(this.#now(), this.#contextRetentionMs()); + this.#scheduleExpiry(); }, at - this.#now()); if (disarm !== undefined) this.#expiry = { at, disarm }; } @@ -767,45 +1022,21 @@ export class TaskScheduler { this.#reconcileScheduled = false; const work = this.#reconcileWork(); if (work) this.#reconcilingWork++; - const cascade = this.#cascadePending; - this.#cascadePending = false; - const checks = [...this.#failFastChecks]; - this.#failFastChecks.clear(); try { - await this.#session.commitWith(async (tx) => { + await this.#session.commitInternal(async (tx) => { if (this.#closing) return; + // A cascade scans the queued inputs, which waits for every unpublished commit: a request made before the scan + // is covered by it, and none is published while the job holds the line after it. + const cascade = this.#cascadePending; const queued = await this.#loadScopes(cascade); - // A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a - // `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. - const marks = new Map(); - const mark = (record: AnyTaskRecord, reason: "restart" | undefined): void => { - const staged = marks.get(record.id); - if (staged !== undefined && staged.reason === undefined) return; - if (record.abortRequested && (record.abortReason === undefined || reason !== undefined)) return; - marks.set(record.id, { record, reason }); - }; - // Every live foreground task below a live owner with cancellation intent, which is its nearest one: a walk down - // from each owner with intent does not enter another, whose own walk covers what is below it. Derived again on - // every pass, as loading edges can reveal a cancelled owner. - for (const id of this.#intent) { - const owner = this.#live.get(id)!; - // Only an owner abandoned after a restart, and nothing else, passes its reason on. - const restart = owner.abortReason === "restart" && !failedOutcome(owner); - for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) { - if (live !== undefined && !node.background) mark(live, restart ? "restart" : undefined); - } - } - for (const id of checks) { - const waiter = this.#live.get(id); - if (waiter?.state.status !== "waiting" || !(await this.#anyFailed(waiter.state.on))) continue; - // Every other live task: the failed one keeps its own outcome. - for (const member of waiter.state.on) { - const record = this.#live.get(member); - if (record !== undefined && !failedOutcome(record)) mark(record, undefined); - } - } + if (cascade) this.#cascadePending = false; + const marks = await this.#marks(); for (const { record, reason } of marks.values()) tx.setTask(withAbortMark(record, reason)); - for (const id of queued) if (this.#belowCancelled({ conversation: id })) await this.#withdrawInputs(tx, id); + const withdrawn = await this.#region( + () => queued.filter((id) => this.#belowCancelled({ conversation: id })), + this.#reloadScopes, + ); + for (const id of withdrawn) await this.#withdrawInputs(tx, id); await this.#finalize(tx); }, this.#context); } catch (error) { @@ -815,7 +1046,87 @@ export class TaskScheduler { } if (work) this.#reconcilingWork--; this.#scheduleWake(); - this.#settleIdle(); + // Its commit's task changes were settled at their publication; a waiter that asked for this pass checks again, and + // what its chain loads resolved can make any scope idle. + if (this.#unknownsChanged) { + this.#settleIdle(undefined); + return; + } + this.#resolveIdleWaiters(undefined); + // A pass checks expiry too, for hosts that decline the expiry timer. + this.#expireContexts(this.#now(), this.#contextRetentionMs()); + this.#scheduleExpiry(); + } + + /** + * The abort marks committed records imply (spec §5.4, §5.5): every live foreground task below a live owner with + * cancellation intent, which is its nearest one (a walk down from each owner with intent does not enter another, + * whose own walk covers what is below it), and every other live task in the `on` of a `failFast` waiter one of whose + * tasks failed. A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades + * a `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. Derived + * again on every pass, as loading edges can reveal a cancelled owner. Takes the `failFast` checks once its decision + * stands. + */ + async #marks(): Promise> { + // Terminal tasks in a checked `on`, read from Storage: each one's outcome is final. + const ended = new Map(); + for (;;) { + const plan = await this.#region(() => this.#planMarks(ended), this.#reloadScopes); + if ("missing" in plan) { + for (const id of plan.missing) + ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null); + continue; + } + for (const id of plan.checked) this.#failFastChecks.delete(id); + return plan.marks; + } + } + + /** `#marks()`' decision region; the `failFast` members it needs from Storage first, when `ended` lacks them. */ + #planMarks(ended: ReadonlyMap): + | { readonly missing: readonly TaskId[] } + | { + readonly marks: Map; + readonly checked: readonly TaskId[]; + } { + const marks = new Map(); + // A mark rewrites the whole record, so the record it copies is read as a task record, keyed: a commit of the task + // that is not published yet makes the region wait and run again, rather than stage a mark over a stale checkpoint. + const mark = (found: AnyTaskRecord, reason: "restart" | undefined): void => { + const record = this.#liveRecord(found.id)!; + const staged = marks.get(record.id); + if (staged !== undefined && staged.reason === undefined) return; + if (record.abortRequested && (record.abortReason === undefined || reason !== undefined)) return; + marks.set(record.id, { record, reason }); + }; + this.#record(INTENT_KEY); + for (const id of this.#intent) { + const owner = this.#liveRecord(id)!; + // Only an owner abandoned after a restart, and nothing else, passes its reason on. + const restart = owner.abortReason === "restart" && !failedOutcome(owner); + for (const { node, live } of this.#walkDown(id, { enter: (below) => !this.#intent.has(below) })) { + if (live !== undefined && !node.background) mark(live, restart ? "restart" : undefined); + } + } + const checked = [...this.#failFastChecks]; + const missing: TaskId[] = []; + for (const id of checked) { + const waiter = this.#liveRecord(id); + if (waiter?.state.status !== "waiting") continue; + let failed = false; + for (const member of waiter.state.on) { + const record = this.#liveRecord(member) ?? ended.get(member); + if (record === undefined) missing.push(member); + else if (record !== null && failedOutcome(record)) failed = true; + } + if (!failed) continue; + // Every other live task: the failed one keeps its own outcome. + for (const member of waiter.state.on) { + const record = this.#liveRecord(member); + if (record !== undefined && !failedOutcome(record)) mark(record, undefined); + } + } + return missing.length > 0 ? { missing } : { marks, checked }; } /** Whether a reconcile pass would have work: a cascade to derive, held outcomes to check, or `failFast` waits. */ @@ -823,55 +1134,86 @@ export class TaskScheduler { return this.#cascadePending || this.#finalizeChecks.size > 0 || this.#failFastChecks.size > 0; } - /** Whether any of `ids` holds or ended with an outcome other than `completed`. */ - async #anyFailed(ids: readonly TaskId[]): Promise { - for (const id of ids) { - const record = this.#live.get(id) ?? (await this.#storage.task(id, this.#context)); - if (record !== undefined && failedOutcome(record)) return true; - } - return false; - } - /** * Write the terminal record of every `completing` task without live ordinary owned work. Finalizing one can free its * owner, so this repeats over the commit's candidates until nothing changes. A held scheduler outcome gets its - * Harness cleanup here. + * Harness cleanup here. In rounds, one decision region each: the holders the tasks of one round free are checked once, + * in the next, so a holder of many tasks that end together is walked once, not once per task. The first round takes + * the `completing` tasks to check once its decision stands. */ async #finalize(tx: Transaction): Promise { - // A worklist: the tasks that started holding, and the holders of work that ended. Finalizing one can free the - // nearest live task above it, which goes on the list. Tasks finalized here have no live work below them, so walks - // down skip them. - // In rounds: the holders the tasks of one round free are checked once, in the next, so a holder of many tasks that - // end together is walked once, not once per task. - let round = [...this.#finalizeChecks]; - this.#finalizeChecks.clear(); const done = new Set(); + // The commit's candidates, kept in step with the terminal records each round stages. const overlay = overlayOf(tx); - while (round.length > 0) { - const next = new Set(); - for (const id of round) { - const record = overlay.tasks.get(id) ?? this.#live.get(id); - if (record?.state.status !== "completing" || done.has(id)) continue; - if (this.#hasOwnedLive(id, overlay, done)) continue; - const outcome = record.state.outcome; - const terminal = withState(record, { status: "terminal", outcome }); + let round: readonly TaskId[] | undefined; + for (;;) { + const ids = round; + const plan = await this.#region( + () => this.#planRound(ids ?? [...this.#finalizeChecks], overlay, done), + this.#reloadScopes, + ); + if (ids === undefined) this.#finalizeChecks.clear(); + for (const { record, terminal } of plan.finalized) { tx.setTask(terminal); - (overlay.tasks as Map).set(id, terminal); - done.add(id); + (overlay.tasks as Map).set(record.id, terminal); + done.add(record.id); + const outcome = (record.state as { readonly outcome: TaskOutcome }).outcome; // REMINDER: only the scheduler writes `faulted` and `orphaned` (spec §5.4); their cleanup waits for this commit. if (outcome.status === "faulted" || outcome.status === "orphaned") { await this.#settleOutcome(tx, record, outcome); } - const above = this.#nearestLiveAbove(record, overlay); - if (above?.state.status === "completing") next.add(above.id); } - round = [...next]; + if (plan.next.length === 0) return; + round = plan.next; } } + /** + * One finalize round's decision: which of `ids` are `completing` without live ordinary owned work below them, in the + * commit's candidates with the round's own finalizations applied in order, and the holders above them to check next. + * Tasks finalized have no live work below them, so walks down skip them. It applies the round's terminal records to + * `overlay` while it decides and takes them back before it returns, so a run again starts from the same candidates. + */ + #planRound( + ids: readonly TaskId[], + overlay: Overlay, + finalized: ReadonlySet, + ): { + readonly finalized: readonly { readonly record: AnyTaskRecord; readonly terminal: AnyTaskRecord }[]; + readonly next: readonly TaskId[]; + } { + const tasks = overlay.tasks as Map; + // The tasks finalized so far, earlier rounds' and this one's, without copying the former. + const round = new Set(); + const done = { has: (id: TaskId) => round.has(id) || finalized.has(id) }; + const ended: { readonly record: AnyTaskRecord; readonly terminal: AnyTaskRecord; readonly staged: boolean }[] = + []; + const next = new Set(); + try { + for (const id of ids) { + const staged = tasks.get(id); + const record = staged ?? this.#liveRecord(id); + if (record?.state.status !== "completing" || done.has(id)) continue; + if (this.#hasOwnedLive(id, overlay, done)) continue; + const terminal = withState(record, { status: "terminal", outcome: record.state.outcome }); + tasks.set(id, terminal); + round.add(id); + ended.push({ record, terminal, staged: staged !== undefined }); + const above = this.#nearestLiveAbove(record, overlay); + if (above?.state.status === "completing") next.add(above.id); + } + } finally { + for (const { record, staged } of ended) { + if (staged) tasks.set(record.id, record); + else tasks.delete(record.id); + } + } + return { finalized: ended, next: [...next] }; + } + /** * Load the owner chains of the tasks in `#unloaded` and, with `queued`, of every conversation with queued inputs, - * on the Session line; returns the latter. Reads committed Storage directly, so it may run inside a commit callback. + * on the Session line; returns the latter. Reads through the line's Storage, so it may run inside a commit callback. */ async #loadScopes(queued: boolean): Promise { for (const id of [...this.#unloaded]) { @@ -879,8 +1221,9 @@ export class TaskScheduler { if (this.#unloaded.has(id)) await this.#loadChain({ task: id }); } if (!queued) return []; + // A scan, which cannot be keyed: it waits for every unpublished commit. const submissions = await scanAll((cursor) => - this.#storage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context), + this.#session.lineStorage.scanSubmissions({ status: "queued" }, SCAN_PAGE_SIZE, cursor, this.#context), ); const inputs = submissions.filter((submission) => submission.type === "input"); const conversations = [...new Set(inputs.map((submission) => submission.conversationId))]; @@ -891,43 +1234,70 @@ export class TaskScheduler { /** * Load the owner edges and task nodes from `start` up to its ownerless root. A known task outside `#unloaded` ends the * walk, as its chain is loaded; the `#unloaded` tasks passed have a loaded chain afterwards. Loading a node can reveal - * a cancelled owner, so it makes the next reconcile derive cascade marks. + * a cancelled owner, so it makes the next reconcile derive cascade marks. The nodes it installs stay until the job's + * commit is published (`#pin()`). */ async #loadChain(start: Up, overlay?: Overlay): Promise { const passed: TaskId[] = []; + let installed = false; let at: Up | undefined = start; - while (at !== undefined) { - if ("task" in at) { - const id: TaskId = at.task; - let node: TaskNode | undefined = this.#live.get(id) ?? this.#settled.get(id); - // Created in this commit: the load from its own record covers what is above it. - if (node === undefined && overlay?.tasks.has(id) === true) break; - if (node !== undefined && !this.#unloaded.has(id)) break; - if (node === undefined) { - const record = await this.#storage.task(id, this.#context); - if (record === undefined) return; - node = nodeOf(record); - if (record.state.status === "terminal") this.#settle(id, node); - // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. - this.#scheduleReconcile(); + try { + while (at !== undefined) { + if ("task" in at) { + const id: TaskId = at.task; + let node: TaskNode | undefined = this.#live.get(id) ?? this.#settled.get(id); + // Created in this commit: the load from its own record covers what is above it. + if (node === undefined && overlay?.tasks.has(id) === true) break; + if (node !== undefined && !this.#unloaded.has(id)) break; + if (node === undefined) { + // Through the line's Storage: a task whose creation or end is not published yet waits for it. + const record = await this.#session.lineStorage.task(id, this.#context); + if (record === undefined) return; + node = nodeOf(record); + if (record.state.status === "terminal") { + this.#settle(id, node); + this.#pin(id); + installed = true; + } + // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. + this.#scheduleReconcile(); + } + passed.push(id); + at = parentOf(node); + } else { + let edge = this.#edge(at.conversation, overlay); + if (edge === undefined) { + // Through the line's Storage: a conversation whose creation is not published yet waits for it, so a missing + // record is never mistaken for an ownerless one. + const record = await this.#session.lineStorage.conversation(at.conversation, this.#context); + edge = record?.owner?.taskId ?? null; + this.#setEdge(at.conversation, edge); + this.#pin(conversationNode(at.conversation)); + installed = true; + // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. + this.#scheduleReconcile(); + } + at = edge === null ? undefined : { task: edge }; } - passed.push(id); - at = parentOf(node); - } else { - let edge = this.#edge(at.conversation, overlay); - if (edge === undefined) { - const record = await this.#storage.conversation(at.conversation, this.#context); - edge = record?.owner?.taskId ?? null; - this.#setEdge(at.conversation, edge); - // A loaded node can reveal a cancelling owner: the next reconcile derives marks again. - this.#scheduleReconcile(); - } - at = edge === null ? undefined : { task: edge }; + } + } finally { + // `#idle()` counts a live task whose chain is not loaded as inside every scope it may be in: what this load + // resolved, also when it stopped early, can make such a scope idle without a task change below it. + if (passed.some((id) => this.#live.has(id)) || (installed && this.#liveUnloaded())) { + this.#unknownsChanged = true; + // A waiter such a task held is checked again at the pass's end, also when this load was not a reconcile's. + this.#scheduleReconcile(); } } for (const id of passed) this.#unloaded.delete(id); } + /** Whether a live task's chain is not loaded yet: one `#idle()` counts as inside every scope it may be in. */ + #liveUnloaded(): boolean { + for (const id of this.#unloaded) if (this.#live.has(id)) return true; + return false; + } + #setEdge(conversationId: ConversationId, owner: TaskId | null): void { this.#edges.set(conversationId, owner); // The conversation now hangs from its owner, or is a root; with nothing below it, it waits for the sweep. @@ -975,7 +1345,8 @@ export class TaskScheduler { */ #sweep(): void { for (const node of this.#dropQueue) { - if (this.#below.has(node)) continue; + // Queued again once its pins go. + if (this.#below.has(node) || this.#pins.has(node)) continue; if (typeof node === "string") { this.#edges.delete(conversationOf(node)); this.#roots.delete(conversationOf(node)); @@ -987,6 +1358,70 @@ export class TaskScheduler { this.#dropQueue.clear(); } + /** + * Run `decide`, a decision region (group commit §6): synchronous, reading scheduler memory and recording the keys it + * reads, staging nothing and changing no scheduler state. When a key it read has an unpublished writer, its result, + * or its error, is dropped: once that writer is published, `refresh` reloads what the region needs and it runs + * again. A waiting job holds the line, so unpublished writes only shrink, and the runs converge. What a region read + * that had no unpublished writer gets none for the rest of the job, so its result stands while the job applies it. + */ + async #region(decide: () => T, refresh?: () => Promise): Promise { + for (;;) { + const keys = new Set(); + this.#reading = keys; + let result: { readonly value: T } | { readonly error: unknown }; + try { + result = { value: decide() }; + } catch (error) { + result = { error }; + } finally { + this.#reading = undefined; + } + const gates: Promise[] = []; + for (const key of keys) { + const gate = this.#session.touch(key); + if (gate !== undefined) gates.push(gate); + } + if (gates.length === 0) { + if ("error" in result) throw result.error; + return result.value; + } + await Promise.all(gates); + await refresh?.(); + } + } + + /** Load the chains a region walks, again after it waited: what was published meanwhile may list new work. */ + readonly #reloadScopes = (): Promise => this.#loadScopes(false); + + /** In a decision region, note that it reads `key`; anywhere else, nothing. */ + #record(key: string): void { + this.#reading?.add(key); + } + + /** A live task's committed record, read by a decision region. */ + #liveRecord(id: TaskId): AnyTaskRecord | undefined { + this.#record(taskKey(id)); + return this.#live.get(id); + } + + /** + * Keep `node`, which a chain load of the job on the line installed, until that job's commit is published or fails: + * work it creates below the node needs it, and the sweep would drop it as unneeded before then. + */ + #pin(node: Node): void { + this.#pins.set(node, (this.#pins.get(node) ?? 0) + 1); + this.#session.onJobEnd(() => { + const count = this.#pins.get(node)! - 1; + if (count > 0) { + this.#pins.set(node, count); + return; + } + this.#pins.delete(node); + this.#dropIfUnneeded(node); + }); + } + /** * Bring the scheduler's sets in line with a task's committed record. A task that ended may free the nearest live task * above it: one holding its outcome finalizes, an abort-marked one runs its abort handler; its waiters may run. @@ -1088,10 +1523,11 @@ export class TaskScheduler { * at or below a background task holds nothing above it. */ #nearestLiveAbove(node: TaskNode, overlay?: Overlay): AnyTaskRecord | undefined { + this.#record(INTENT_KEY); for (let current = node; !current.background; ) { const owner = current.owner ?? this.#edge(current.conversationId, overlay); if (owner === undefined || owner === null) return undefined; - const record = overlay?.tasks.get(owner) ?? this.#live.get(owner); + const record = overlay?.tasks.get(owner) ?? this.#liveRecord(owner); if (record !== undefined && record.state.status !== "terminal") return record; const above = record ?? this.#settled.get(owner); if (above === undefined) return undefined; @@ -1111,7 +1547,9 @@ export class TaskScheduler { /** Owner task of a conversation, `null` when ownerless, `undefined` while not loaded. */ #edge(id: ConversationId, overlay?: Overlay): TaskId | null | undefined { - return overlay?.edges.has(id) === true ? overlay.edges.get(id) : this.#edges.get(id); + if (overlay?.edges.has(id) === true) return overlay.edges.get(id); + this.#record(conversationKey(id)); + return this.#edges.get(id); } #node(id: TaskId, overlay?: Overlay): TaskNode | undefined { @@ -1159,6 +1597,7 @@ export class TaskScheduler { const links = overlay === undefined ? undefined : this.#links(overlay); const pending: Node[] = [start]; for (let at = pending.pop(); at !== undefined; at = pending.pop()) { + this.#record(ownedKey(at)); for (const children of [this.#below.get(at), links?.get(at)]) { for (const child of children ?? []) { if (typeof child === "string") { @@ -1213,7 +1652,7 @@ export class TaskScheduler { */ #ownedLive( owner: TaskId, - options: { readonly overlay?: Overlay; readonly first?: boolean; readonly done?: ReadonlySet } = {}, + options: { readonly overlay?: Overlay; readonly first?: boolean; readonly done?: Finalized } = {}, ): TaskId[] { const { overlay, first = false, done } = options; const found: TaskId[] = []; @@ -1229,7 +1668,7 @@ export class TaskScheduler { return found; } - #hasOwnedLive(owner: TaskId, overlay?: Overlay, done?: ReadonlySet): boolean { + #hasOwnedLive(owner: TaskId, overlay?: Overlay, done?: Finalized): boolean { const options = { first: true, ...(overlay === undefined ? {} : { overlay }), @@ -1244,6 +1683,7 @@ export class TaskScheduler { * edge is not loaded. */ #inScope(start: Up, scope: Scope, crossBackground = false): boolean | undefined { + this.#record(INTENT_KEY); for (const step of this.#above(start)) { if ("unknown" in step) return undefined; if ("conversation" in step) { @@ -1265,6 +1705,7 @@ export class TaskScheduler { /** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */ #cancellingOwner(start: Up): AnyTaskRecord | undefined { + this.#record(INTENT_KEY); // Nothing cancels, the common case: no walk. if (this.#intent.size === 0) return undefined; for (const step of this.#above(start)) { @@ -1278,7 +1719,7 @@ export class TaskScheduler { } /** - * Count a task in or out of `pi.durable.tasks.active`. Commit observers count inside Storage's `adopt`, which must not + * Count a task in or out of `pi.durable.tasks.active`. Commit observers count at a commit's publication, which must not * throw, so a throwing adapter is reported. */ #countActive(delta: 1 | -1, kind: string): void { @@ -1308,6 +1749,7 @@ export class TaskScheduler { this.#taskWaiters.rejectAll(error); this.#idleWaiters.rejectAll(error); this.#contexts.clear(); + this.#idleOrder.clear(); this.#expiry?.disarm(); this.#expiry = undefined; this.#timer?.disarm(); @@ -1454,47 +1896,36 @@ export class TaskScheduler { const faulted: { readonly record: AnyTaskRecord; readonly message: string }[] = []; let abandoned: TaskId[] = []; try { - await this.#session.commitWith(async (tx) => { + await this.#session.commitInternal(async (tx) => { if (!this.#enabled || this.#closing) return; - for (const id of this.#abandoned) { - const record = this.#live.get(id); - if (record !== undefined && !record.abortRequested) tx.setTask(withAbortMark(record, "restart")); - } if (this.#abandoned.size > 0) { - abandoned = [...this.#abandoned]; + const ids = [...this.#abandoned]; + const unmarked = await this.#region(() => + ids + .map((id) => this.#liveRecord(id)) + .filter((record) => record !== undefined && !record.abortRequested), + ); + for (const record of unmarked) tx.setTask(withAbortMark(record!, "restart")); + abandoned = ids; return; } await this.#loadScopes(false); // Every pass picks up sleeps whose time has come: a pass started by any change may be the first to run after // a deadline, and the wake it schedules no longer keeps a timer for a due sleep. this.#wakeDue(); + const plan = await this.#region(() => this.#planReservation(), this.#reloadScopes); + for (const id of plan.dropped) this.#runnable.delete(id); + // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule + // the cascade, which may not have run yet. + if (plan.cascade) { + this.#cascadePending = true; + this.#scheduleReconcile(); + } // Taken once per pass, and only when some task is a candidate. let snapshot: RegistrySnapshot | undefined; - for (const id of [...this.#runnable]) { - const record = this.#live.get(id); - if (record === undefined || record.state.status === "completing" || this.#invocations.has(id)) { - this.#runnable.delete(id); - continue; - } - // It returns once what it waits for ends, its record changes, or the work below it ends. - if (this.#waitingOn(record, true).length > 0) { - this.#runnable.delete(id); - continue; - } - // It returns at the first pass after its time comes, which the timer, `resume()`, or any change starts. - if (this.#notDue(record)) { - this.#runnable.delete(id); - continue; - } - const runnable = record as RunnableTaskRecord; + for (const record of plan.ready) { + const id = record.id; const mode = record.abortRequested ? "abort" : "run"; - // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule - // the cascade, which may not have run yet. - if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) { - this.#cascadePending = true; - this.#scheduleReconcile(); - continue; - } // Work cut off `restartLimit` times in a row would run forever: an abort handler, or a phase of a kind that // does not end itself at the limit, ends the task `faulted` instead. // The settings are read only for a task that was cut off at all. @@ -1512,7 +1943,7 @@ export class TaskScheduler { continue; } snapshot ??= this.#registry.snapshot(); - const resolution = this.#resolve(runnable, snapshot); + const resolution = this.#resolve(record, snapshot); if (resolution.kind === "blocked") { // An abandoned task waits for its definition, so its abort handler can clean up. if (mode === "abort" && record.abortReason === undefined) { @@ -1520,7 +1951,7 @@ export class TaskScheduler { } continue; } - if (resolution.record !== runnable || record.state.status !== "running") { + if (resolution.record !== record || record.state.status !== "running") { tx.setTask( withState(resolution.record, { status: "running", @@ -1531,7 +1962,7 @@ export class TaskScheduler { // Registered on the line, so marks and later reservations see it and close joins it; `#end` makes it a // candidate again. this.#runnable.delete(id); - const invocation = this.#createInvocation(runnable, mode, installerOf(snapshot, resolution.task)); + const invocation = this.#createInvocation(record, mode, installerOf(snapshot, resolution.task)); reservations.push({ invocation, task: resolution.task, snapshot }); } }, this.#context); @@ -1556,6 +1987,46 @@ export class TaskScheduler { return reservations; } + /** + * A reservation pass's decision region: the candidates it drops (gone, holding, waiting, asleep, or with an + * invocation), whether one waits for a cascade, and the runnable records to resolve. Iterating the candidates is not + * keyed: a pass that misses a task a pending commit makes runnable only delays it, as does skipping one with an + * invocation without reading its record, or a waiter's check whether what it waits on is still live. + */ + #planReservation(): { + readonly dropped: readonly TaskId[]; + readonly cascade: boolean; + readonly ready: readonly RunnableTaskRecord[]; + } { + const dropped: TaskId[] = []; + const ready: RunnableTaskRecord[] = []; + let cascade = false; + for (const id of this.#runnable) { + if (this.#invocations.has(id)) { + dropped.push(id); + continue; + } + const record = this.#liveRecord(id); + // It returns once what it waits for ends, its record changes, or the work below it ends; a sleep at the first pass + // after its time comes, which the timer, `resume()`, or any change starts. + if ( + record === undefined || + record.state.status === "completing" || + this.#waitingOn(record, true).length > 0 || + this.#notDue(record) + ) { + dropped.push(id); + continue; + } + if (!record.abortRequested && !record.background && this.#belowCancelled(parentOf(record))) { + cascade = true; + continue; + } + ready.push(record as RunnableTaskRecord); + } + return { dropped, cascade, ready }; + } + /** * Live tasks a task waits for before its next invocation: its live ordinary owned work when abort-marked, since * abort runs bottom-up, otherwise the live part of the `on` of a wait. @@ -1603,7 +2074,7 @@ export class TaskScheduler { /** Sizes of the ownership indexes, for tests that bound the scheduler's memory. */ get indexSizes(): Readonly< - Record<"live" | "settled" | "edges" | "below" | "roots" | "unloaded" | "dropQueue", number> + Record<"live" | "settled" | "edges" | "below" | "roots" | "unloaded" | "dropQueue" | "pins", number> > { let below = 0; for (const children of this.#below.values()) below += children.size; @@ -1615,6 +2086,33 @@ export class TaskScheduler { roots: this.#roots.size, unloaded: this.#unloaded.size, dropQueue: this.#dropQueue.size, + pins: this.#pins.size, + }; + } + + /** + * The kept contexts' idle state, for tests (a diagnostic, computed on demand): how many are kept and how many are + * idle, how many disagree with a full recompute (an idle period without `#idle()`, or `#idle()` without one; an + * expiry order that differs from the idle periods, or is not sorted), and how many idle checks covered every scope. + */ + get idleState(): Readonly> { + let mismatches = 0; + let idle = 0; + for (const [conversationId, kept] of this.#contexts) { + if (kept.idleSince !== undefined) idle++; + if ((kept.idleSince !== undefined) !== this.#idle(conversationId)) mismatches++; + if (this.#idleOrder.get(conversationId) !== kept.idleSince) mismatches++; + } + let previous = Number.NEGATIVE_INFINITY; + for (const [conversationId, idleSince] of this.#idleOrder) { + if (idleSince < previous || this.#contexts.get(conversationId)?.idleSince !== idleSince) mismatches++; + previous = idleSince; + } + return { + contexts: this.#contexts.size, + idleContexts: idle, + idleMismatches: mismatches, + fullIdleChecks: this.#fullIdleChecks, }; } @@ -1626,6 +2124,7 @@ export class TaskScheduler { async inspect( snapshot: RegistrySnapshot, ): Promise<{ scheduling: HarnessInspection["scheduling"]; tasks: TaskInspection[] }> { + await this.#published(); await this.#loadScopes(false); const tasks: TaskInspection[] = []; for (const record of this.#live.values()) tasks.push({ record, state: this.#inspectTask(record, snapshot) }); @@ -1695,9 +2194,9 @@ export class TaskScheduler { const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined as ReportedTask }; const phase: Phase = { snapshot: () => state.snapshot, task: () => state.task, agent: undefined }; const runtime = this.#runtime(invocation, phase); - let current = await this.#step(invocation, (tx, next) => - this.#decide(tx, next, undefined, state, invocation.extension, invocation.context), - ); + const progress = (tx: Transaction, current: ErasedRunningTask) => + this.#progress(tx, current, state, invocation.extension, invocation.context); + let current = await this.#step(invocation, (task) => this.#rules(task, undefined), progress); while (current !== undefined && !this.#closing) { const task = current; current = await this.#phase(invocation, task, async () => { @@ -1711,9 +2210,7 @@ export class TaskScheduler { } catch (error) { previous = { checkpoint, failure: { error } }; } - return this.#step(invocation, (tx, next) => - this.#decide(tx, next, previous, state, invocation.extension, invocation.context), - ); + return this.#step(invocation, (next) => this.#rules(next, previous), progress); }); } } @@ -1785,53 +2282,56 @@ export class TaskScheduler { } /** - * Precedence rules for a run invocation, on the line. Rules 1 (terminal, `completing`, `waiting`, or `sleeping`) and 2 - * (closing) are applied by `#step`. Returns whether the invocation continues with the next phase. + * Precedence rules for a run invocation, a decision region of its step. Rules 1 (terminal, `completing`, `waiting`, or + * `sleeping`) and 2 (closing) are applied by `#step`. Progress (rule 5) is decided by `#progress()` once the decision + * stands, as it takes a registry snapshot. */ - #decide( - tx: Transaction, - current: ErasedRunningTask, - previous: PhaseResult | undefined, - state: { task: AnyTask; snapshot: RegistrySnapshot; reported: ReportedTask }, - extension: string | undefined, - context: Context, - ): Decision { + #rules(current: ErasedRunningTask, previous: PhaseResult | undefined): StepPlan { // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone. - if (current.abortRequested) return false; + if (current.abortRequested) return END; // An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back. - if (!current.background && this.#belowCancelled(parentOf(current))) { - this.#cascadePending = true; - this.#scheduleReconcile(); - return false; - } - if (previous === undefined) return true; + if (!current.background && this.#belowCancelled(parentOf(current))) return CASCADE; + if (previous === undefined) return CONTINUE; // 4. uncaught error. - if (previous.failure !== undefined) return { fault: previous.failure.error }; + if (previous.failure !== undefined) return { kind: "fault", error: previous.failure.error }; // 6. no durable progress. if (jsonEqual(current.state.checkpoint, previous.checkpoint)) { const message = `Task ${current.kind} phase ${previous.checkpoint.phase} returned without durable progress`; - return { fault: new Error(message) }; + return { kind: "fault", error: new Error(message) }; } - // 5. progress: refresh the snapshot; hand over to a replacement definition that can take the task. + return PROGRESS; + } + + /** + * Rule 5, progress: refresh the snapshot; hand over to a replacement definition that can take the task, writing it + * back as `pending`, or keep running under the old one, reported once per resolved definition. Returns whether the + * invocation continues. + */ + #progress( + tx: Transaction, + current: ErasedRunningTask, + state: { task: AnyTask; snapshot: RegistrySnapshot; reported: ReportedTask }, + extension: string | undefined, + context: Context, + ): boolean { state.snapshot = this.#registry.snapshot(); const next = state.snapshot.task(current.kind); - if (next !== state.task) { - if (next !== undefined && canReserve(next, current)) { - tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint })); - return false; - } - if (state.reported === undefined || state.reported.task !== next) { - state.reported = { task: next }; - this.#report( - new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), - { - kind: "task_outdated", - ...this.#aboutTask({ ...recordTask(current), extension }), - reason: outdatedReason(next, current), - }, - context, - ); - } + if (next === state.task) return true; + if (next !== undefined && canReserve(next, current)) { + tx.setTask(withState(current, { status: "pending", checkpoint: current.state.checkpoint })); + return false; + } + if (state.reported === undefined || state.reported.task !== next) { + state.reported = { task: next }; + this.#report( + new Error(`Task ${current.id} keeps running under its old ${current.kind} definition`), + { + kind: "task_outdated", + ...this.#aboutTask({ ...recordTask(current), extension }), + reason: outdatedReason(next, current), + }, + context, + ); } return true; } @@ -1855,31 +2355,44 @@ export class TaskScheduler { failure = { error }; } const message = `Abort handler of task ${invocation.taskId} returned without a terminal outcome`; - await this.#step(invocation, () => ({ fault: failure?.error ?? new Error(message) })); + await this.#step(invocation, () => ({ kind: "fault", error: failure?.error ?? new Error(message) })); }); } /** - * One synchronous decision on the Session line. A task that is no longer running (rule 1: terminal, `completing`, - * `waiting`, or `sleeping`) or a closing Harness (rule 2) ends the invocation without a write; otherwise `decide` may - * stage a write and returns whether the invocation continues. Ending happens inside the callback, before a fault's - * Harness cleanup. A rejected step, such as admission after close, also ends the invocation. + * One decision on the Session line, before a phase: a decision region (`#region()`) that reads the task's record. A + * task that is no longer running (rule 1: terminal, `completing`, `waiting`, or `sleeping`) or a closing Harness (rule + * 2) ends the invocation without a write; otherwise `plan` decides, and a decision that stands is applied: the + * invocation continues, with `progress` for rule 5, or ends, with a fault written. Ending happens inside the callback, + * before a fault's Harness cleanup. A rejected step, such as admission after close, also ends the invocation. Its + * outcome goes to the invocation only, so it is a scheduler commit (`commitInternal()`). */ async #step( invocation: Invocation, - decide: (tx: Transaction, current: ErasedRunningTask) => Decision, + plan: (current: ErasedRunningTask) => StepPlan, + progress?: (tx: Transaction, current: ErasedRunningTask) => boolean, ): Promise { let fault: { readonly error: unknown } | undefined; try { - const continued = await this.#session.commitWith(async (tx) => { - const found = this.#live.get(invocation.taskId); - const current = found?.state.status === "running" ? (found as ErasedRunningTask) : undefined; - const decision = current !== undefined && !this.#closing ? decide(tx, current) : false; - if (decision === true) return current; + const continued = await this.#session.commitInternal(async (tx) => { + const decided = await this.#region(() => { + const found = this.#liveRecord(invocation.taskId); + if (found?.state.status !== "running" || this.#closing) return { plan: END }; + const current = found as ErasedRunningTask; + return { current, plan: plan(current) }; + }); + const { current } = decided; + const step = decided.plan; + if (step.kind === "continue") return current; + if (step.kind === "progress" && progress!(tx, current!)) return current; this.#end(invocation); - if (decision !== false) { - fault = { error: decision.fault }; - const message = errorMessage(decision.fault); + if (step.kind === "cascade") { + this.#cascadePending = true; + this.#scheduleReconcile(); + } + if (step.kind === "fault") { + fault = { error: step.error }; + const message = errorMessage(step.error); invocation.fault = { status: "faulted", error: { message } }; await this.#terminate(tx, current!, invocation.fault); } @@ -1908,7 +2421,7 @@ export class TaskScheduler { */ async #terminate(tx: Transaction, record: AnyTaskRecord, outcome: SchedulerOutcome): Promise { await this.#loadScopes(false); - if (this.#hasOwnedLive(record.id, overlayOf(tx))) { + if (await this.#region(() => this.#hasOwnedLive(record.id, overlayOf(tx)), this.#reloadScopes)) { tx.setTask(withState(record, { status: "completing", outcome })); return; } @@ -1933,7 +2446,7 @@ export class TaskScheduler { const overlay = overlayOf(tx); await this.#loadScopes(false); for (const record of overlay.tasks.values()) await this.#loadChain(parentOf(record), overlay); - if (this.#hasOwnedLive(current.id, overlay)) { + if (await this.#region(() => this.#hasOwnedLive(current.id, overlayOf(tx)), this.#reloadScopes)) { tx.setTask(withState(progressed(current), { status: "completing", outcome: next.outcome })); return; } @@ -1960,37 +2473,71 @@ export class TaskScheduler { policy: string, ): Promise { if (invocation.mode === "abort") throw new Error(`Abort handler of task ${current.id} cannot wait`); - const overlay = overlayOf(tx); + // Owned work is found by walks down, which need the chains of every live task loaded. + await this.#loadScopes(false); + // Tasks in `on` that are not live, read from Storage: terminal, or missing. + const ended = new Map(); + for (;;) { + const checked = await this.#region( + () => this.#checkWait(current, on, policy, overlayOf(tx), ended), + this.#reloadScopes, + ); + if (checked === undefined) return; + if ("missing" in checked) { + for (const id of checked.missing) + ended.set(id, (await this.#session.lineStorage.task(id, this.#context)) ?? null); + continue; + } + throw checked.error; + } + } + + /** + * `#validateWait()`'s decision region: why the wait is refused, `undefined` when it may be stored, or the tasks in + * `on` it needs from Storage first, when `ended` lacks them. + */ + #checkWait( + current: ErasedRunningTask, + on: readonly TaskId[], + policy: string, + overlay: Overlay, + ended: ReadonlyMap, + ): { readonly missing: readonly TaskId[] } | { readonly error: Error } | undefined { const targets: TaskId[] = []; + const missing: TaskId[] = []; for (const id of on) { - const member = overlay.tasks.get(id) ?? this.#live.get(id) ?? (await this.#storage.task(id, this.#context)); - if (member === undefined) throw new Error(`Task ${id} does not exist`); + const member = overlay.tasks.get(id) ?? this.#liveRecord(id) ?? ended.get(id); + if (member === undefined) { + missing.push(id); + continue; + } + if (member === null) return { error: new Error(`Task ${id} does not exist`) }; if (policy === "failFast" && member.owner !== current.id) { - throw new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`); + return { error: new Error(`Task ${current.id} can wait failFast only on tasks it owns; ${id} is not one`) }; } // A live child already holds the waiter (task-owned tasks are never background): waiting on it adds no edge. A // terminal task has no edges. if (member.state.status !== "terminal" && member.owner !== current.id) targets.push(id); } - if (targets.length === 0) return; - const cycle = await this.#waitCycle(current.id, targets, overlay); - if (cycle !== undefined) { - throw new Error( + if (missing.length > 0) return { missing }; + if (targets.length === 0) return undefined; + const cycle = this.#waitCycle(current.id, targets, overlay); + if (cycle === undefined) return undefined; + return { + error: new Error( `Task ${current.id} cannot wait on ${cycle[0]}: ${[current.id, ...cycle].join(" -> ")} is a wait cycle`, - ); - } + ), + }; } /** * The path from a member of `on` back to `task` in the wait-for graph, if the wait would close a cycle (spec §5.1). * An edge X -> Y means X cannot settle before Y: Y is in the `on` of waiting X that is not abort-marked, or Y is live - * ordinary owned work of X, - * which holds X as `completing`. Each task is visited once and each subtree walked once, so a check is bounded by the - * live work it reaches. + * ordinary owned work of X, which holds X as `completing`. Each task is visited once and each subtree walked once, + * so a check is bounded by the live work it reaches. Part of a decision region: it records every record it reads. + * Owned work is found by walks down, which need the chains of every live task loaded. */ - async #waitCycle(task: TaskId, on: readonly TaskId[], overlay: Overlay): Promise { - // Owned work is found by walks down, which need the chains of every live task loaded. - await this.#loadScopes(false); + #waitCycle(task: TaskId, on: readonly TaskId[], overlay: Overlay): TaskId[] | undefined { const from = new Map(); const walked = new Set(); const pending: TaskId[] = []; @@ -2007,7 +2554,7 @@ export class TaskScheduler { for (const id of on) reach(id, undefined); for (let id = pending.pop(); id !== undefined; id = pending.pop()) { if (id === task) return path(id); - const record = overlay.tasks.get(id) ?? this.#live.get(id); + const record = overlay.tasks.get(id) ?? this.#liveRecord(id); if (record === undefined || record.state.status === "terminal") continue; // An abort-marked task waits only for its owned work, as abort runs bottom-up (`#waitingOn`). if (record.state.status === "waiting" && !record.abortRequested) { @@ -2046,6 +2593,8 @@ export class TaskScheduler { /** No live non-background task in the scope; a task whose owner edges are not loaded yet counts as inside. */ #idle(conversationId: ConversationId | undefined): boolean { const scope: Scope = conversationId === undefined ? { roots: true } : { conversation: conversationId }; + // In a decision region, global idle reads the roots it starts from. + if (conversationId === undefined) this.#record(ROOTS_KEY); // Tasks whose chain is not loaded are not listed below their scope yet; one counts as inside unless its walk up // decides otherwise. for (const id of this.#unloaded) { @@ -2197,7 +2746,7 @@ export class TaskScheduler { )) as ErasedRuntime["snapshotAsOf"], getTask: ((id: TaskId, context: Context) => this.#read(invocation, () => - this.#session.readOnLine(() => this.#storage.task(id, context)), + this.#session.readOnLine((storage) => storage.task(id, context)), )) as ErasedRuntime["getTask"], waitForTask: ((id: TaskId, context: Context) => this.#read(invocation, () => @@ -2210,11 +2759,15 @@ export class TaskScheduler { if (invocation.ended) throw endedError(invocation); context.abortSignal?.throwIfAborted(); // Every live task is listed under its parent node, its owner task when it has one. - const owned: AnyTaskRecord[] = []; - for (const node of this.#below.get(invocation.taskId) ?? []) { - const record = typeof node === "string" ? undefined : this.#live.get(node); - if (record?.owner === invocation.taskId) owned.push(record); - } + const owned = await this.#region(() => { + this.#record(ownedKey(invocation.taskId)); + const records: AnyTaskRecord[] = []; + for (const node of this.#below.get(invocation.taskId) ?? []) { + const record = typeof node === "string" ? undefined : this.#liveRecord(node); + if (record?.owner === invocation.taskId) records.push(record); + } + return records; + }); // Copies, as from storage: the scheduler reads its own. return owned .sort((a, b) => a.id - b.id) @@ -2224,7 +2777,7 @@ export class TaskScheduler { abortOwned: (id, context) => this.#read(invocation, async () => { const bound = withAbortSignal(invocation.controller.signal, context); - const record = await this.#session.readOnLine(() => this.#storage.task(id, bound)); + const record = await this.#session.readOnLine((storage) => storage.task(id, bound)); if (record?.owner !== invocation.taskId) { throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`); } @@ -2234,10 +2787,10 @@ export class TaskScheduler { }), outcomes: ((ids: readonly TaskId[], context: Context) => this.#read(invocation, () => - this.#session.readOnLine(async () => { + this.#session.readOnLine(async (storage) => { const outcomes: TaskOutcome[] = []; for (const id of ids) { - const state = (await this.#storage.task(id, context))?.state; + const state = (await storage.task(id, context))?.state; if (state?.status !== "terminal") throw new Error(`Task ${id} is not terminal`); outcomes.push(state.outcome); } @@ -2263,27 +2816,24 @@ export class TaskScheduler { ? [undefined, args[0] as EntryId, args[1] as Context] : [args[0] as { readonly kind: string }, args[1] as EntryId, args[2] as Context]; return this.#read(invocation, async () => { - const found = await this.#session.readOnLine(() => - this.#storage.entry(invocation.conversationId, id, context), - ); - const entry: EntryRecord | undefined = found?.entry; + // An entry of the conversation's kept context range is committed and visible: no Storage read. + const kept = this.#contexts.get(invocation.conversationId); + const entry = + (kept === undefined ? undefined : rangeEntry(kept.range, id)) ?? + (await this.#session.readOnLine((storage) => storage.entry(invocation.conversationId, id, context))) + ?.entry; return token === undefined || entry?.kind === token.kind ? entry : undefined; }); }) as ErasedRuntime["entry"], entries: (query, limit, cursor, context) => this.#read(invocation, () => - this.#session.readOnLine(() => - this.#storage.scanEntries( - conversationEntries(invocation.conversationId, query), - limit, - cursor, - context, - ), + this.#session.readOnLine((storage) => + storage.scanEntries(conversationEntries(invocation.conversationId, query), limit, cursor, context), ), ), context: (conversationId, context, options) => this.#read(invocation, async () => { - const { view, range } = await readContextFrom( + const read = await readContextFrom( this.#session, this.#storage, conversationId, @@ -2291,6 +2841,14 @@ export class TaskScheduler { options?.at, this.#contexts.get(conversationId)?.range, ); + const { view } = read; + // Checked again here, synchronously up to the install below: a commit published since the read returned + // was not seen by `#advanceContexts`, so a range that missed it must not be advanced. + const range = + read.range?.current === true && + this.#session.knownLatestEntryId(conversationId) !== read.range.bounds.tail + ? { ...read.range, current: false } + : read.range; // Keep it unless the invocation ended or a concurrent read already kept a newer range. const kept = this.#contexts.get(conversationId); if ( @@ -2302,8 +2860,16 @@ export class TaskScheduler { // A read of another, idle conversation starts or continues its retention period. if (!this.#idle(conversationId)) { this.#contexts.set(conversationId, { range, idleSince: undefined }); + this.#idleOrder.delete(conversationId); } else if (this.#contextRetentionMs() > 0) { - this.#contexts.set(conversationId, { range, idleSince: kept?.idleSince ?? this.#now() }); + // An idle period that already started keeps its expiry and its place; a new one starts now. + const idleSince = kept?.idleSince ?? this.#now(); + const expiresAt = + kept?.idleSince !== undefined && kept.expiresAt !== undefined + ? kept.expiresAt + : idleSince + this.#contextRetentionMs(); + this.#contexts.set(conversationId, { range, idleSince, expiresAt }); + if (kept?.idleSince === undefined) this.#markIdle(conversationId, idleSince); this.#scheduleExpiry(); } } @@ -2343,6 +2909,15 @@ export class TaskScheduler { }; } + /** + * On the line: wait until every commit admitted before is published, for a decision that reads what cannot be keyed, + * every live task (open, inspection) or every task a conversation abort reaches. + */ + async #published(): Promise { + const gate = this.#session.touch(ALL_KEYS); + if (gate !== undefined) await gate; + } + /** Run a committed-state read unless the invocation has ended. */ async #read(invocation: Invocation, read: () => Promise): Promise { if (invocation.ended) throw endedError(invocation); @@ -2361,13 +2936,18 @@ export class TaskScheduler { async (tx) => { if (invocation.ended) throw endedError(invocation); if (this.#closing) throw closedError(this.#session); - const found = this.#live.get(invocation.taskId); - if (found === undefined) throw new Error(`Task ${invocation.taskId} is terminal`); - if (found.state.status !== "running") throw new Error(`Task ${invocation.taskId} is ${found.state.status}`); - const current = found as ErasedRunningTask; - if (invocation.mode === "run" && current.abortRequested) { - throw new Error(`Task ${invocation.taskId} has a durable abort mark`); - } + const current = await this.#region(() => { + const found = this.#liveRecord(invocation.taskId); + if (found === undefined) throw new Error(`Task ${invocation.taskId} is terminal`); + if (found.state.status !== "running") + throw new Error(`Task ${invocation.taskId} is ${found.state.status}`); + if (invocation.mode === "run" && found.abortRequested) { + throw new Error(`Task ${invocation.taskId} has a durable abort mark`); + } + return found as ErasedRunningTask; + }); + // The step that ends it may have run while the region waited. + if (invocation.ended) throw endedError(invocation); return change(tx, current); }, context, @@ -2449,6 +3029,19 @@ function cancellationIntent(record: AnyTaskRecord): boolean { return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record)); } +/** + * Whether replacing `previous` (absent: a new task) with `next` changes the cancellation intent set or what it means: a + * mark set, upgraded, or cleared, or a held or ended outcome other than `completed`. + */ +function intentChanges(previous: AnyTaskRecord | undefined, next: AnyTaskRecord): boolean { + return ( + (previous?.abortRequested ?? false) !== next.abortRequested || + previous?.abortReason !== next.abortReason || + (previous !== undefined && failedOutcome(previous)) !== failedOutcome(next) || + (previous !== undefined && cancellationIntent(previous)) !== cancellationIntent(next) + ); +} + /** Whether the record holds or ends with an outcome other than `completed`. */ function failedOutcome(record: AnyTaskRecord): boolean { const state = record.state; diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 66f116af3..bac428f44 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -8,7 +8,7 @@ import { } from "@earendil-works/pi-telemetry"; import { ConversationBusy, ConversationIdle, ReadAfterWrite } from "../errors.ts"; import type { ObservedCommit, SessionImpl } from "../session/session.ts"; -import type { ConversationId, JsonObject, Storage, SubmissionId, SubmissionRecord, Tx } from "../types.ts"; +import type { ConversationId, JsonObject, SubmissionId, SubmissionRecord, Tx } from "../types.ts"; import { startRun } from "./generation.ts"; import { appendInput, @@ -34,7 +34,6 @@ type Admission = { readonly id: SubmissionId; readonly status: "queued" | "place export class Submissions { readonly #telemetry: TelemetryContext; readonly #session: SessionImpl; - readonly #storage: Storage; readonly #now: () => number; /** Read at each admission, on the Session line. */ readonly #queueModes: () => QueueModes; @@ -45,7 +44,6 @@ export class Submissions { constructor( session: SessionImpl, - storage: Storage, now: () => number, queueModes: () => QueueModes, resume: () => void, @@ -53,7 +51,6 @@ export class Submissions { ) { this.#telemetry = telemetry; this.#session = session; - this.#storage = storage; this.#now = now; this.#queueModes = queueModes; this.#resume = resume; @@ -123,12 +120,12 @@ export class Submissions { /** Handle for an existing submission, or `undefined`. */ async get(id: SubmissionId, context: Context): Promise { - const record = await this.#session.readOnLine(() => this.#storage.submission(id, context)); + const record = await this.#session.readOnLine((storage) => storage.submission(id, context)); return record === undefined ? undefined : new SubmissionHandle(record.id, this); } async status(id: SubmissionId, context: Context): Promise { - const record = await this.#session.readOnLine(() => this.#storage.submission(id, context)); + const record = await this.#session.readOnLine((storage) => storage.submission(id, context)); if (record === undefined) throw new Error(`Submission ${id} does not exist`); return record; } @@ -136,8 +133,8 @@ export class Submissions { async wait(id: SubmissionId, context: Context): Promise { this.#resume(); // Check and register on the line so no settling publication falls between them. - const found = await this.#session.readOnLine(async () => { - const record = await this.#storage.submission(id, context); + const found = await this.#session.readOnLine(async (storage) => { + const record = await storage.submission(id, context); if (record === undefined) throw new Error(`Submission ${id} does not exist`); if (isSettled(record)) return { promise: Promise.resolve(record) }; // Close rejects registered waiters synchronously and may begin during the read. diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index 2e0b78875..c7fbcb824 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,16 +1,10 @@ -import { - type Context, - type JsonValue, - Watch, - type WatchCall, - type WatchEnd, - type WatchHandle, -} from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchCall, type WatchHandle } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { + closeEndOf, + closeEnds, contained, - ENDED, guardObserver, type ObservedCommit, observation, @@ -93,14 +87,12 @@ const SCAN_PAGE_SIZE = 256; */ export class TaskGraphView { readonly #session: SessionImpl; - readonly #storage: Storage; readonly #blocking: TaskGraphBlocking; #mount: Mount | undefined; #closed = false; - constructor(session: SessionImpl, storage: Storage, blocking: TaskGraphBlocking) { + constructor(session: SessionImpl, blocking: TaskGraphBlocking) { this.#session = session; - this.#storage = storage; this.#blocking = blocking; session.followCommits((publication, context) => { // A commit held in memory changes documents only, which the graph does not show. @@ -110,11 +102,10 @@ export class TaskGraphView { blocking.subscribeBlocked(() => this.#refreshBlocked()); session.subscribeClose(() => { this.#closed = true; - const failure = this.#session.failure; - const end: WatchEnd = failure === undefined ? ENDED : { kind: "failed", error: failure.error }; + const ends = closeEnds(this.#session.failure); for (const observer of [...(this.#mount?.observers ?? [])]) { contained( - () => observer.end(end), + () => observer.end(closeEndOf(observer, ends)), (error) => session.report(error), ); } @@ -160,8 +151,11 @@ export class TaskGraphView { create: (value: TaskGraph, release: () => void) => O, context: Context, ): Promise<{ observer: O; detach: () => void }> { - return this.#session.readOnLine(async () => { - const mount = this.#mount ?? { ...(await this.#build(context)), observers: new Set>() }; + return this.#session.readOnLine(async (storage) => { + const mount = this.#mount ?? { + ...(await this.#build(storage, context)), + observers: new Set>(), + }; const detach = (): void => { mount.observers.delete(observer); if (mount.observers.size === 0 && this.#mount === mount) this.#mount = undefined; @@ -197,19 +191,18 @@ export class TaskGraphView { .catch(() => {}); } - async #build(context: Context): Promise> { + /** Read the graph on the line, through the line's `storage`. */ + async #build(storage: Storage, context: Context): Promise> { const tasks: Record = {}; const live = new Map(); const records: AnyTaskRecord[] = []; for (const status of LIVE_STATUSES) { - records.push( - ...(await scanAll((cursor) => this.#storage.scanTasks({ status }, SCAN_PAGE_SIZE, cursor, context))), - ); + records.push(...(await scanAll((cursor) => storage.scanTasks({ status }, SCAN_PAGE_SIZE, cursor, context)))); } records.sort((a, b) => a.id - b.id); for (const record of records) { const owned = await scanAll((cursor) => - this.#storage.scanConversations({ ownerTaskId: record.id }, SCAN_PAGE_SIZE, cursor, context), + storage.scanConversations({ ownerTaskId: record.id }, SCAN_PAGE_SIZE, cursor, context), ); const conversations = owned.map((conversation) => conversation.id).sort((a, b) => a - b); const subject = blockedSubject(record); diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index bdc806aa9..dc9e72ca2 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -11,7 +11,8 @@ import { } from "@earendil-works/chord/output"; import type { ImageContent, TextContent, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai"; import { validateToolArguments } from "@earendil-works/pi-ai/utils/validation"; -import { Compile } from "typebox/compile"; +import { compile, type Validator } from "@earendil-works/pi-schema"; +import { formats } from "@earendil-works/pi-schema/formats"; import { defineDocFamily } from "../documents.ts"; import { AssistantEntry, ToolResultEntry } from "../entries.ts"; import type { ExecutionEnv } from "../env/index.ts"; @@ -177,13 +178,21 @@ export const ToolTask = defineTask { - const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); - if (slot !== undefined) slot.status = "running"; - // A nested slot shows the arguments the call runs with, which repair, hooks, and coercion may have changed. - // Compared with the input, which the slot got at admission: reading the draft would track every leaf. - const changed = task.input.kind === "nested" && !jsonEqual(task.input.call.arguments as JsonValue, final); - if (changed && slot !== undefined && "parentTaskId" in slot) slot.arguments = final; + // The slot's status and arguments are presentation, stored like progress (`PROGRESS_COMMIT`); the intent is + // durable before the tool runs. Parallel calls of one round then share no immediate write to `pi.live`. + await runtime.commit( + async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot !== undefined) slot.status = "running"; + // A nested slot shows the arguments the call runs with, which repair, hooks, and coercion may have changed. + // Assigned whole: an equal value records no change, and a rerun corrects what an earlier attempt showed. + if (slot !== undefined && "parentTaskId" in slot) slot.arguments = final; + return undefined; + }, + context, + PROGRESS_COMMIT, + ); + await runtime.commit(() => { const intent = { phase: "execute", arguments: final, replay: tool.replay ?? "unsafe" } as const; return { status: "running", checkpoint: intent }; }, context); @@ -197,10 +206,15 @@ export const ToolTask = defineTask { const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); - if (slot !== undefined) clearProgress(slot); + if (slot !== undefined) { + clearProgress(slot); + slot.status = "running"; + if ("parentTaskId" in slot) slot.arguments = args; + } return undefined; }, context); return run(runtime, task.input, call, tool, args, context); @@ -757,7 +771,7 @@ async function afterHarnessResult( return { ...rest, ...value, output: bounded.content, diagnostics: [...(rest.diagnostics ?? []), ...harness] }; } -const structuredValidators = new WeakMap>(); +const structuredValidators = new WeakMap(); /** Why a result's `structuredOutput` breaks the tool's contract, or `undefined` when it keeps it. */ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResult): string | undefined { @@ -773,12 +787,17 @@ function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResu } let validator = structuredValidators.get(schema); if (validator === undefined) { - validator = Compile(schema); + validator = compile(schema, { formats, unresolvedRefs: "reject" }); structuredValidators.set(schema, validator); } - if (validator.Check(value)) return undefined; - const [first] = validator.Errors(value); - const path = first?.instancePath.replace(/^\//, "").replace(/\//g, ".") || "root"; + if (validator.check(value)) return undefined; + const [first] = validator.errors(value); + const path = + first?.path + .slice(1) + .split("/") + .map((segment) => segment.replace(/~1/g, "/").replace(/~0/g, "~")) + .join(".") || "root"; return `Tool ${tool.name} returned structuredOutput that does not match its schema: ${path}: ${first?.message ?? "invalid"}`; } diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index d7f4a05ec..e312758b9 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -1,20 +1,21 @@ import type { Context, JsonValue, WatchHandle } from "@earendil-works/chord"; import type { + Api, AssistantMessage, CacheRetention, ClientIdentity, Message, + Model, Models, ModelThinkingLevel, - Static, Tool, ToolCall, ToolResultMessage, Transport, - TSchema, Usage, UserMessage, } from "@earendil-works/pi-ai"; +import type { Static } from "@earendil-works/pi-schema"; import type { TelemetryContext } from "@earendil-works/pi-telemetry"; import type { ExecutionEnv, ShellOutputSkip, ShellOutputWindow } from "../env/index.ts"; import type { @@ -331,12 +332,22 @@ export interface ToolExecutionApi extend ): Promise; } +/** + * The arguments type of a tool with parameter schema `TParameters`: pi-schema's `Static` (the builders' + * type, TypeBox's for the common TypeBox kinds, else `unknown`), and `any` for `ToolRegistration`. + */ +export type ToolArgs = IsAny extends true ? any : Static; + +// `0 extends 1 & T` is resolved too early for a `T` constrained to `object`; this form waits for the argument. +type IsAny = boolean extends (T extends never ? true : false) ? true : false; + /** * Executable tool registered in a registry. Only pi-ai `Tool` fields enter the transcript. `args` are typed by - * `parameters`, which the Harness validates them against before `execute()`; `defineTool()` infers both generics. + * `parameters` (any JSON Schema object; see {@link ToolArgs}), which the Harness validates them against before + * `execute()`; `defineTool()` infers both generics. */ export type ToolRegistration< - TParameters extends TSchema = TSchema, + TParameters extends object = object, TDetails extends JsonValue = JsonValue, > = Tool & { /** Whether an interrupted execution may rerun on recovery. Default `unsafe`. */ @@ -348,12 +359,12 @@ export type ToolRegistration< * pure and must not mutate `args`: it runs again when a call is retried before its intent is recorded. Its result is * still validated against `parameters`. */ - prepareArguments?(args: unknown): Static; + prepareArguments?(args: unknown): ToolArgs; /** * Schema of `structuredOutput`, which programs that call the tool receive. Omitted: they receive the result's bounded * output, success or error, as `NestedToolExecutionResult.structuredOutput` describes. */ - readonly structuredOutputSchema?: TSchema; + readonly structuredOutputSchema?: object; /** * Who may call the tool: the model, other tools through `executeTool()`, or both (default). A code mode tool is * `["model"]`, so scripts cannot start scripts; tools behind code mode, such as MCP tools, can be `["tools"]`, so the @@ -371,7 +382,7 @@ export type ToolRegistration< readonly retain?: "head" | "tail"; }; execute( - args: Static, + args: ToolArgs, api: ToolExecutionApi, context: Context, ): Promise>; @@ -970,6 +981,16 @@ export interface GenerationHooks { api: HookApi, context: Context, ): HookResult<{ readonly messages: readonly Message[] }>; + /** + * After pi-ai builds the provider request body, before it is sent; the result replaces the body for that request only. + * Runs again whenever pi-ai rebuilds the body, such as a retry within one attempt. The body is in the provider API's + * own format (`request.model.api`), so a hook checks the API before changing it. + */ + beforeProviderRequest( + request: { readonly model: Model; readonly payload: unknown }, + api: HookApi, + context: Context, + ): HookResult<{ readonly payload: unknown }>; /** Every terminal provider message, before classification. */ afterResponse(message: AssistantMessage, api: HookApi, context: Context): void | Promise; /** A final answer; the first `continue` appends a user message and continues the run. */ diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index d2999c47f..130d1c459 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -10,8 +10,9 @@ import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; import { idFromNumber } from "../ids.ts"; import { + closeEndOf, + closeEnds, contained, - ENDED, guardObserver, type ObservedCommit, observation, @@ -144,13 +145,11 @@ function resolveWindow(window: ConversationWindow | undefined): WindowBounds { */ export class ConversationViews { readonly #session: SessionImpl; - readonly #storage: Storage; readonly #mounts = new Map(); #closed = false; - constructor(session: SessionImpl, storage: Storage) { + constructor(session: SessionImpl) { this.#session = session; - this.#storage = storage; VIEWS.set(session, this); session.followCommits((publication, context) => { for (const [id, mount] of this.#mounts) @@ -158,12 +157,11 @@ export class ConversationViews { }); session.subscribeClose(() => { this.#closed = true; - const failure = this.#session.failure; - const end: WatchEnd = failure === undefined ? ENDED : { kind: "failed", error: failure.error }; + const ends = closeEnds(this.#session.failure); for (const mount of this.#mounts.values()) for (const windowed of [...mount.observers]) { contained( - () => windowed.observer.end(end), + () => windowed.observer.end(closeEndOf(windowed.observer, ends)), (error) => this.report(error), ); } @@ -236,9 +234,9 @@ export class ConversationViews { context: Context, ): Promise<{ observer: O; detach: () => void }> { const bounds = resolveWindow(window); - return this.#session.readOnLine(async () => { - const mount = this.#mounts.get(id) ?? (await this.#build(id, context)); - const { entries, before } = await loadWindow(this.#storage, id, bounds, context); + return this.#session.readOnLine(async (storage) => { + const mount = this.#mounts.get(id) ?? (await this.#build(storage, id, context)); + const { entries, before } = await loadWindow(storage, id, bounds, context); const value = view(mount.conversation, entries, before, mount.docs); let windowed: Windowed | undefined; const detach = (): void => { @@ -256,8 +254,9 @@ export class ConversationViews { }); } - async #build(id: ConversationId, context: Context): Promise { - const conversation = await this.#storage.conversation(id, context); + /** Hydrate a mount on the line, through the line's `storage`. */ + async #build(storage: Storage, id: ConversationId, context: Context): Promise { + const conversation = await storage.conversation(id, context); if (conversation === undefined) throw new Error(`Conversation ${id} does not exist`); const docs: Record = {}; const incarnations = new Map(); diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 4b4671129..9d35a9378 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -54,6 +54,7 @@ export { } from "./harness/live.ts"; export { ProviderDoc, type ProviderState } from "./harness/provider.ts"; export { createRegistry } from "./harness/registry.ts"; +export { registryReader } from "./harness/registry-reader.ts"; export type { BlockedReason } from "./harness/scheduler.ts"; export type { TaskGraph, TaskGraphNode, TaskGraphState, TaskGraphWatch } from "./harness/task-graph.ts"; export { DURABLE_TELEMETRY_SCHEMA } from "./harness/telemetry.ts"; @@ -146,6 +147,7 @@ export type { CommitOptions, CommitPersistence, CommitPublication, + CommitResult, CommonDocDefinition, ContextEdit, ConversationDocFamilyToken, diff --git a/packages/durable/src/search.ts b/packages/durable/src/search.ts index eb89772a9..9dc6ed5ec 100644 --- a/packages/durable/src/search.ts +++ b/packages/durable/src/search.ts @@ -3,7 +3,115 @@ * ranking: durable keeps its own copy of what it needs from the coding agent. */ -export type SearchDocument = { readonly name: string; readonly text: string }; +/** A tool as the ranker sees it: its name, its text, and the text of its namespace. */ +export type SearchDocument = { + readonly name: string; + /** Description and parameter names and descriptions; not the name, which the ranker reads from `name`. */ + readonly text: string; + /** Name, description, and instructions of the tool's namespace. */ + readonly namespaceText?: string; +}; + +/** + * Okapi BM25 (k1 1.2, b 0.75) over `text`, with the name and the namespace as separate fields: the names of the best + * matches, best first; ties keep document order. + * - A tool that matches more of the query's terms, in any field, ranks above one that matches fewer, whatever their + * scores: for `screenshot window`, a tool whose description has both words beats `window` whose name has one. + * - A query term in the name scores more than any number of mentions in the text. + * - A tool whose whole local name (after the last `__`) is in the query gets a bonus worth one more match of every + * query term, so among tools matching as many terms, `peekaboo app` finds `mcp__peekaboo__app` over tools that + * only mention the words. + * - A query that is exactly a tool's name, or its name after a `__`, ranks that tool first, before coverage. + * + * Only `text` is length-normalized, so a long name or long namespace instructions do not lower a tool's score. + */ +export function rankDocuments(query: string, documents: readonly SearchDocument[], limit: number): string[] { + const queryTerms = new Set(tokenize(query)); + if (queryTerms.size === 0 || documents.length === 0 || limit <= 0) return []; + const indexed = documents.map(indexDocument); + const averageLength = indexed.reduce((sum, document) => sum + document.textLength, 0) / indexed.length || 1; + const idf = new Map( + [...queryTerms].map((term) => { + const frequency = indexed.filter( + (document) => + document.nameTerms.has(term) || document.textCounts.has(term) || document.namespaceTerms.has(term), + ).length; + return [term, Math.log(1 + (indexed.length - frequency + 0.5) / (frequency + 0.5))] as const; + }), + ); + const fullNameBonus = [...idf.values()].reduce((sum, value) => sum + value, 0); + const exactQuery = normalizeName(query); + const matches: { name: string; score: number; covered: number; exact: boolean }[] = []; + for (const document of indexed) { + const norm = K1 * (1 - B + (B * document.textLength) / averageLength); + let score = 0; + let covered = 0; + for (const term of queryTerms) { + const count = document.textCounts.get(term) ?? 0; + const weight = + (document.nameTerms.has(term) ? NAME_WEIGHT : 0) + + (document.namespaceTerms.has(term) ? NAMESPACE_WEIGHT : 0) + + (count * (K1 + 1)) / (count + norm); + if (weight > 0) covered++; + score += idf.get(term)! * weight; + } + if (score === 0) continue; + const local = document.localNameTerms; + if (local.length > 0 && local.every((term) => queryTerms.has(term))) score += fullNameBonus; + matches.push({ name: document.name, score, covered, exact: isExactNameQuery(exactQuery, document.name) }); + } + return matches + .sort( + (left, right) => + Number(right.exact) - Number(left.exact) || right.covered - left.covered || right.score - left.score, + ) + .slice(0, limit) + .map((match) => match.name); +} + +const K1 = 1.2; + +const B = 0.75; + +/** + * A query term in the name counts this many times its IDF. The text part of a term is at most `K1 + 1` (2.2) times + * its IDF and about 1 for a single mention, so a name match beats any number of mentions in the text. + */ +const NAME_WEIGHT = 3; + +/** + * A query term in the namespace text counts this many times its IDF. Every tool of the namespace shares it, so it helps + * find the namespace but not the tool in it. + */ +const NAMESPACE_WEIGHT = 0.5; + +/** A document split into the fields the ranker weighs differently. */ +type IndexedDocument = { + name: string; + /** Terms of the name: `mcp`, `app`, `thread`, `read` for `mcp__app__thread_read`. */ + nameTerms: ReadonlySet; + /** Terms of the name after its last `__`, the part models say: `thread`, `read`. */ + localNameTerms: readonly string[]; + textCounts: ReadonlyMap; + textLength: number; + namespaceTerms: ReadonlySet; +}; + +function indexDocument(document: SearchDocument): IndexedDocument { + const textCounts = new Map(); + for (const term of tokenize(document.text)) textCounts.set(term, (textCounts.get(term) ?? 0) + 1); + const segments = document.name.split("__"); + return { + name: document.name, + nameTerms: new Set(tokenize(document.name)), + localNameTerms: [...new Set(tokenize(segments[segments.length - 1]!))], + textCounts, + textLength: [...textCounts.values()].reduce((sum, count) => sum + count, 0), + namespaceTerms: new Set(tokenize(document.namespaceText ?? "")), + }; +} + +const wordSegmenter = new Intl.Segmenter(undefined, { granularity: "word" }); const STOP_WORDS: ReadonlySet = new Set([ "a", @@ -29,6 +137,24 @@ const STOP_WORDS: ReadonlySet = new Set([ "with", ]); +/** Unicode word segmentation with identifier boundaries and English stop words/singular forms. */ +export function tokenize(text: string): string[] { + const normalized = text + .normalize("NFC") + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") + .toLowerCase() + .replace(/[^\p{L}\p{M}\p{N}]+/gu, " "); + // The segmenter costs about 15 times a split. Plain ASCII splits into the same words at spaces, so only other + // text, such as Chinese written without spaces, goes through it. + const words = /^[a-z0-9 ]*$/.test(normalized) + ? normalized.split(" ") + : Array.from(wordSegmenter.segment(normalized), (segment) => (segment.isWordLike ? segment.segment : "")); + return words + .filter((word) => word.length > 0 && !STOP_WORDS.has(word)) + .map((word) => (/^[a-z0-9]+$/.test(word) ? stem(word) : word)); +} + /** Naive singular form, so `issues` matches `issue` and `searches` matches `search`. */ function stem(term: string): string { if (term.length > 4 && term.endsWith("ies")) return `${term.slice(0, -3)}y`; @@ -37,49 +163,19 @@ function stem(term: string): string { return term; } -/** Lowercase terms, split at camelCase boundaries and non-alphanumerics, without stop words. */ -export function tokenize(text: string): string[] { - return text - .replace(/([a-z0-9])([A-Z])/g, "$1 $2") - .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2") +/** Lowercase with characters invalid in script identifiers as `_`, so `dev-radius` matches `dev_radius`. */ +function normalizeName(name: string): string { + return name + .trim() .toLowerCase() - .split(/[^a-z0-9]+/) - .filter((term) => term.length > 0 && !STOP_WORDS.has(term)) - .map(stem); + .replace(/[^a-z0-9_$]/g, "_"); } -/** Okapi BM25 (k1 1.2, b 0.75): the names of the best matches, best first; ties keep document order. */ -export function rankDocuments(query: string, documents: readonly SearchDocument[], limit: number): string[] { - const k1 = 1.2; - const b = 0.75; - const queryTerms = [...new Set(tokenize(query))]; - if (queryTerms.length === 0 || documents.length === 0 || limit <= 0) return []; - const termCounts = documents.map((document) => { - const counts = new Map(); - for (const term of tokenize(document.text)) counts.set(term, (counts.get(term) ?? 0) + 1); - return counts; - }); - const lengths = termCounts.map((counts) => [...counts.values()].reduce((sum, count) => sum + count, 0)); - const averageLength = lengths.reduce((sum, length) => sum + length, 0) / documents.length || 1; - const idf = new Map( - queryTerms.map((term) => { - const frequency = termCounts.filter((counts) => counts.has(term)).length; - return [term, Math.log(1 + (documents.length - frequency + 0.5) / (frequency + 0.5))] as const; - }), - ); - const matches: { name: string; score: number }[] = []; - documents.forEach((document, index) => { - let score = 0; - for (const term of queryTerms) { - const count = termCounts[index]!.get(term); - if (!count) continue; - const norm = k1 * (1 - b + (b * lengths[index]!) / averageLength); - score += (idf.get(term) ?? 0) * ((count * (k1 + 1)) / (count + norm)); - } - if (score > 0) matches.push({ name: document.name, score }); - }); - return matches - .sort((left, right) => right.score - left.score) - .slice(0, limit) - .map((match) => match.name); +/** + * Whether the whole query is the tool's name: its full name or a suffix after a `__` separator, so `thread_read` and + * `app__thread_read` both name `mcp__app__thread_read`. + */ +function isExactNameQuery(query: string, name: string): boolean { + const normalized = normalizeName(name); + return normalized === query || normalized.endsWith(`__${query}`); } diff --git a/packages/durable/src/session/cached-storage.ts b/packages/durable/src/session/cached-storage.ts new file mode 100644 index 000000000..5087a2834 --- /dev/null +++ b/packages/durable/src/session/cached-storage.ts @@ -0,0 +1,378 @@ +// Write-through cache of hot Storage record reads for the Session that owns the Storage. +import type { Context, JsonValue } from "@earendil-works/chord"; +import type { + CommitResult, + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../types.ts"; + +type AnyTaskRecord = TaskRecord; +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** + * Storage that serves hot record reads from memory: conversations, tasks, and written submissions, each table bounded + * by a count, the least recently used record leaving first, except live tasks, which are kept until they end, and the + * ended tasks a task waits on, kept until it moved on from its wait: a task's record is read again at each of its state + * changes, a waiter reads the outcomes of what it waited on once it resumes (`runtime.outcomes()`), and both are + * bounded by live work, which the scheduler keeps in memory too. Without that, a burst of more tasks than the count + * would make each of those reads a Storage read on the line. Every commit goes through it, and one process owns a + * Storage, so it applies each batch as it is published and never goes stale: `commit()`'s batches when the Session publishes + * them (`published()`), not at their durability receipts. A read + * result is kept only when no commit published since the read started wrote its record, as the read may not observe + * that commit; versions are kept only for records with a read in flight. Cached values are frozen and shared. Entries, + * scans, and documents pass through: the Session keeps documents and the newest entry per conversation itself, and + * context ranges hold the entries a run reads. + * + * Not for a read-only Session: another process writes its Storage. + */ +export class CachedStorage implements Storage { + readonly #storage: Storage; + readonly #conversations = new Lru(4096); + /** Live (non-terminal) task records, until their task ends. */ + readonly #liveTasks = new Map(); + /** + * Tasks some task waits on, or waited on until its wait ended and it has not moved on since: how many such waits name + * each, and its ended record once it ended. + */ + readonly #awaited = new Map(); + /** Per task that waits, or left its wait and has not changed since: what it waited on, and whether it left. */ + readonly #waits = new Map(); + // Ended task records carry inputs and outcomes, such as a tool call's arguments, so fewer are kept. + readonly #tasks = new Lru(256); + readonly #submissions = new Lru(256); + /** Per record with a read in flight: the reads, and how many published commits wrote it since the first began. */ + readonly #reads = new Map, { count: number; version: number }>(); + + constructor(storage: Storage) { + this.#storage = storage; + } + + /** The backend's own scope for a publication (`Storage.publishing`), when it has one. */ + get publishing(): Storage["publishing"] { + return scopeOf(this.#storage); + } + + /** Pass the batches through; the Session applies each one with `published()` when it publishes it. */ + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + return this.#storage.commit(batches, receive, context); + } + + /** + * Apply a batch `commit()` stored, when the Session publishes it. Its records are frozen in place and kept as + * they are, no copies: the Session's components that follow the publication, such as the scheduler's live tasks, + * hold the same objects, so a live task's record is in memory once. + */ + published(writes: readonly StorageWrite[]): void { + for (const write of writes) { + switch (write.type) { + case "conversation": + this.#wrote(write.value.id); + this.#conversations.set(write.value.id, freeze(write.value)); + break; + case "task": + this.#wrote(write.value.id); + this.#followWait(write.value); + this.#keepTask(freeze(write.value)); + break; + case "submission": + this.#wrote(write.value.id); + this.#submissions.set(write.value.id, freeze(write.value)); + break; + } + } + } + + mintId>(): Promise { + return this.#storage.mintId(); + } + + conversation(id: ConversationId, context: Context): Promise { + const cached = this.#conversations.get(id); + if (cached !== undefined) return Promise.resolve(cached); + return this.#keep( + id, + () => this.#storage.conversation(id, context), + (record) => this.#conversations.set(id, record), + ); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanConversations(query, limit, cursor, context); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + first: EntryId | ConversationId, + second: EntryId | Context, + third?: Context, + ): Promise { + if (third === undefined) return this.#storage.entry(first as EntryId, second as Context); + return this.#storage.entry(first as ConversationId, second as EntryId, third); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#storage.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanEntries(query, limit, cursor, context); + } + + task(id: TaskId, context: Context): Promise { + const cached = this.#liveTasks.get(id) ?? this.#awaited.get(id)?.record ?? this.#tasks.get(id); + if (cached !== undefined) return Promise.resolve(cached); + return this.#keep( + id, + () => this.#storage.task(id, context), + (record) => this.#keepTask(record), + ); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanTasks(query, limit, cursor, context); + } + + /** Cached as written only: this process writes every submission it settles, and a read one is usually settled. */ + submission(id: SubmissionId, context: Context): Promise { + const cached = this.#submissions.get(id); + return cached === undefined ? this.#storage.submission(id, context) : Promise.resolve(cached); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanSubmissions(query, limit, cursor, context); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + return this.#storage.submissionByRequest(conversationId, requestId, context); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#storage.findDocument(address, at, context); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + return this.#storage.document(id, at, context); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#storage.scanDocuments(query, limit, cursor, context); + } + + close(context: Context): Promise { + return this.#storage.close(context); + } + + /** Records kept per table, for tests that bound memory. */ + get sizes(): { + readonly conversations: number; + readonly tasks: number; + readonly liveTasks: number; + readonly awaited: number; + readonly waits: number; + readonly submissions: number; + } { + return { + conversations: this.#conversations.size, + tasks: this.#tasks.size, + liveTasks: this.#liveTasks.size, + awaited: this.#awaited.size, + waits: this.#waits.size, + submissions: this.#submissions.size, + }; + } + + /** Keep a frozen task record: a live one until its task ends, an ended one in the bounded table. */ + #keepTask(record: AnyTaskRecord): void { + if (record.state.status === "terminal") { + this.#liveTasks.delete(record.id); + const awaited = this.#awaited.get(record.id); + if (awaited !== undefined) awaited.record = record; + else this.#tasks.set(record.id, record); + return; + } + this.#tasks.delete(record.id); + this.#liveTasks.set(record.id, record); + } + + /** + * Follow a task's waits from its written record: a wait holds the ended records of its `on` until the task, after + * leaving the wait, changes again (or ends), as the phase that resumes reads their outcomes before its next commit. + */ + #followWait(record: AnyTaskRecord): void { + const wait = this.#waits.get(record.id); + if (record.state.status === "waiting") { + if (wait !== undefined) this.#release(wait.on); + const on = record.state.on; + this.#waits.set(record.id, { on, left: false }); + for (const id of on) { + let awaited = this.#awaited.get(id); + if (awaited === undefined) { + // One that ended already moves over from the bounded table. + const ended = this.#tasks.get(id); + this.#tasks.delete(id); + awaited = ended === undefined ? { count: 0 } : { count: 0, record: ended }; + this.#awaited.set(id, awaited); + } + awaited.count++; + } + return; + } + if (wait === undefined) return; + if (!wait.left && record.state.status !== "terminal") { + wait.left = true; + return; + } + this.#waits.delete(record.id); + this.#release(wait.on); + } + + /** A wait on `on` is over: what no other wait names goes back to the bounded table. */ + #release(on: readonly TaskId[]): void { + for (const id of on) { + const awaited = this.#awaited.get(id)!; + if (--awaited.count > 0) continue; + this.#awaited.delete(id); + if (awaited.record !== undefined) this.#tasks.set(id, awaited.record); + } + } + + /** A published commit wrote record `id`: a read of it in flight may have missed that. */ + #wrote(id: Id): void { + const reads = this.#reads.get(id); + if (reads !== undefined) reads.version++; + } + + /** Resolve with a read's result, frozen; keep a found record unless a commit published since wrote it. */ + async #keep( + id: Id, + read: () => Promise, + keep: (value: T) => void, + ): Promise { + let reads = this.#reads.get(id); + if (reads === undefined) { + reads = { count: 0, version: 0 }; + this.#reads.set(id, reads); + } + reads.count++; + const version = reads.version; + let value: T | undefined; + try { + value = await read(); + } finally { + if (--reads.count === 0) this.#reads.delete(id); + } + if (value === undefined) return undefined; + freeze(value); + if (reads.version === version) keep(value); + return value; + } + + /** Records with a read in flight, for tests that bound memory. */ + get readsInFlight(): number { + return this.#reads.size; + } +} + +/** `storage`'s publication scope (`Storage.publishing`), for a wrapper to expose as its own; none when it has none. */ +export function scopeOf(storage: Storage): Storage["publishing"] { + if (storage.publishing === undefined) return undefined; + return (publish, context) => storage.publishing!(publish, context); +} + +/** A Map bounded to `capacity` entries that drops the least recently used one. */ +class Lru { + readonly #map = new Map(); + readonly #capacity: number; + + constructor(capacity: number) { + this.#capacity = capacity; + } + + get(key: K): V | undefined { + const value = this.#map.get(key); + if (value === undefined) return undefined; + this.#map.delete(key); + this.#map.set(key, value); + return value; + } + + get size(): number { + return this.#map.size; + } + + delete(key: K): void { + this.#map.delete(key); + } + + set(key: K, value: V): void { + this.#map.delete(key); + this.#map.set(key, value); + if (this.#map.size > this.#capacity) this.#map.delete(this.#map.keys().next().value!); + } +} + +function freeze(value: T): T { + if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) freeze(child); + return Object.freeze(value); +} diff --git a/packages/durable/src/session/conversation-tails.ts b/packages/durable/src/session/conversation-tails.ts new file mode 100644 index 000000000..2064f77b0 --- /dev/null +++ b/packages/durable/src/session/conversation-tails.ts @@ -0,0 +1,115 @@ +// The newest entry and newest head marker of each recently used conversation, kept current from the Session's commits. +import type { ConversationId, EntryId, EntryRecord, StorageWrite } from "../types.ts"; + +/** An entry that carries a `head`: the marker of a context range. */ +export type HeadMarker = EntryRecord & { readonly head: EntryId }; + +/** Conversations whose newest entry ID is kept: IDs only, so many. */ +const LATEST_CAPACITY = 4096; + +/** Conversations whose newest head marker is kept: whole entries (a compaction summary is one), so fewer. */ +const MARKER_CAPACITY = 256; + +/** + * The newest entry ID and newest head marker per conversation, for the Session that owns its Storage: every entry is + * written by one of its commits, so `apply` keeps both current, and a value read from Storage stays valid until the + * next entry of that conversation. Read only on the Session line after touching the conversation's appends, so no + * commit that appends to it is unpublished and a value read from Storage can be kept. The newest entry visible from a conversation is its own newest, or for a fork without entries + * of its own the parent's entry at the fork point: IDs grow, so a conversation's own new entry is always the newest. + * Each map is bounded by a count, the least recently used conversation leaving first. Kept markers are frozen: entry + * records are immutable (`Tx.appendEntry()`). + */ +export class ConversationTails { + /** Newest entry ID per conversation, `null` for none. */ + readonly #latest = new Map(); + /** Newest head marker per conversation, `null` for none. */ + readonly #markers = new Map(); + + /** The newest entry ID visible from `conversationId`, read with `read` when not known. */ + async latest( + conversationId: ConversationId, + read: () => Promise, + ): Promise { + const known = use(this.#latest, conversationId); + if (known !== undefined) return known ?? undefined; + const latest = await read(); + put(this.#latest, conversationId, latest ?? null, LATEST_CAPACITY); + return latest; + } + + /** + * The newest visible head marker at or before `atOrBefore` (any when `undefined`), read with `read` when not known. + * The newest marker answers every cutoff at or after it, and no marker answers every cutoff. + */ + async marker( + conversationId: ConversationId, + atOrBefore: EntryId | undefined, + read: () => Promise, + ): Promise { + const marker = use(this.#markers, conversationId); + if (marker === null) return undefined; + if (marker !== undefined && (atOrBefore === undefined || atOrBefore >= marker.id)) return marker; + const latest = this.#latest.get(conversationId); + const found = await read(); + // Only a read that no newer entry can change: no cutoff, or one at or after the newest entry. + if (atOrBefore === undefined || (latest !== undefined && (latest ?? 0) <= atOrBefore)) { + put(this.#markers, conversationId, found === undefined ? null : freeze(found), MARKER_CAPACITY); + } + return found; + } + + /** The newest entry ID of `conversationId` if known, without reading: `null` for none, `undefined` when unknown. */ + peek(conversationId: ConversationId): EntryId | null | undefined { + return this.#latest.get(conversationId); + } + + /** At a commit's publication: apply a durable batch. */ + apply(writes: readonly StorageWrite[]): void { + for (const write of writes) { + if (write.type === "conversation") { + // A new root conversation sees no entries; a fork sees its parent's up to the fork point. + if (write.value.parent !== undefined) continue; + put(this.#latest, write.value.id, null, LATEST_CAPACITY); + put(this.#markers, write.value.id, null, MARKER_CAPACITY); + } else if (write.type === "entry") { + const entry = write.value; + put(this.#latest, entry.conversationId, entry.id, LATEST_CAPACITY); + if (entry.head !== undefined) { + put(this.#markers, entry.conversationId, freeze(entry as HeadMarker), MARKER_CAPACITY); + } + } + } + } + + /** How many conversations have their newest entry kept, for tests that bound memory. */ + get size(): number { + return this.#latest.size; + } + + /** How many conversations have their newest head marker kept, for tests that bound memory. */ + get markers(): number { + return this.#markers.size; + } +} + +/** `map`'s value for `key`, made the most recently used. */ +function use(map: Map, key: K): V | undefined { + const value = map.get(key); + if (value === undefined) return undefined; + map.delete(key); + map.set(key, value); + return value; +} + +/** Set `key` as the most recently used, dropping the least recently used beyond `capacity`. */ +function put(map: Map, key: K, value: V, capacity: number): void { + map.delete(key); + map.set(key, value); + if (map.size > capacity) map.delete(map.keys().next().value!); +} + +function freeze(value: T): T { + if (value === null || typeof value !== "object" || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) freeze(child); + return Object.freeze(value); +} diff --git a/packages/durable/src/session/keys.ts b/packages/durable/src/session/keys.ts new file mode 100644 index 000000000..85fa8f2d3 --- /dev/null +++ b/packages/durable/src/session/keys.ts @@ -0,0 +1,274 @@ +// The keys of the Session line: what a commit writes and a job reads, so a read of an unpublished write waits for it. +import type { Context, JsonValue } from "@earendil-works/chord"; +import { addressId } from "../documents.ts"; +import type { + CommitResult, + ConversationId, + ConversationQuery, + ConversationRecord, + Cursor, + DocumentAddress, + DocumentId, + DocumentPoint, + DocumentQuery, + DocumentRecord, + EntryId, + EntryQuery, + EntryRecord, + Id, + Page, + Seq, + Storage, + StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, + TaskId, + TaskQuery, + TaskRecord, +} from "../types.ts"; + +type StoredEntry = { readonly entry: EntryRecord; readonly commitSeq: Seq }; + +/** A task record. */ +export function taskKey(id: TaskId): string { + return `task:${id}`; +} + +/** A conversation record. */ +export function conversationKey(id: ConversationId): string { + return `conv:${id}`; +} + +/** An entry record. */ +export function entryKey(id: EntryId): string { + return `entry:${id}`; +} + +/** The newest entries of a conversation: appended blind, read by scans, the newest entry, and head markers. */ +export function appendKey(id: ConversationId): string { + return `entry-append:${id}`; +} + +/** A submission record. */ +export function submissionKey(id: SubmissionId): string { + return `sub:${id}`; +} + +/** The submission a conversation's request ID names. */ +export function requestKey(conversationId: ConversationId, requestId: string): string { + return `req:${conversationId}:${requestId}`; +} + +/** A document address (`addressId()`): its current incarnation, value, and held changes. */ +export function documentKey(addressId: string): string { + return `doc:${addressId}`; +} + +/** A document incarnation, for reads by its ID. */ +export function incarnationKey(id: DocumentId): string { + return `docid:${id}`; +} + +/** The incarnations alive in a scope: created and retired blind, read by scans and the task-document index. */ +export function scopeKey(scope: DocumentRecord["scope"]): string { + switch (scope.kind) { + case "session": + return "docscope:session"; + case "conversation": + return `docscope:c${scope.conversationId}`; + case "task": + return `docscope:t${scope.taskId}`; + } +} + +/** + * A node of the scheduler's ownership tree (a task ID, or `c`): the live work listed below it. Written + * by task creation and a task becoming terminal for every node from the task's parent up to and including the nearest + * live task or the root conversation; read by every walk that expands the node. + */ +export function ownedKey(node: TaskId | string): string { + return `owned:${node}`; +} + +/** The ownerless conversations with live work below them, which global idle walks from. */ +export const ROOTS_KEY = "roots"; + +/** The live tasks with cancellation intent: abort marks, and held outcomes other than `completed`. */ +export const INTENT_KEY = "intent"; + +/** Every unpublished write: what a read that cannot be keyed, such as a scan, touches. */ +export const ALL_KEYS = "*"; + +/** The keys of the table records in `writes`; document writes are keyed by the transaction, which knows addresses. */ +export function tableKeys(writes: readonly StorageWrite[], keys: string[]): void { + for (const write of writes) { + switch (write.type) { + case "conversation": + keys.push(conversationKey(write.value.id)); + break; + case "entry": + keys.push(entryKey(write.value.id), appendKey(write.value.conversationId)); + break; + case "task": + keys.push(taskKey(write.value.id)); + break; + case "submission": + keys.push(submissionKey(write.value.id)); + if (write.value.requestId !== undefined) { + keys.push(requestKey(write.value.conversationId, write.value.requestId)); + } + break; + } + } +} + +/** + * The Storage a job on the line reads through: every read first touches its key (`touch`), which waits while the key + * has an unpublished writer, so the read sees what the line admitted before it. Scans that cannot be keyed touch + * `ALL_KEYS`. Commits and `close()` pass through; the line never commits through it. + */ +export class GatedStorage implements Storage { + readonly #storage: Storage; + readonly #touch: (key: string) => Promise | undefined; + + constructor(storage: Storage, touch: (key: string) => Promise | undefined) { + this.#storage = storage; + this.#touch = touch; + } + + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + return this.#storage.commit(batches, receive, context); + } + + mintId>(): Promise { + return this.#storage.mintId(); + } + + conversation(id: ConversationId, context: Context): Promise { + return this.#after(conversationKey(id), () => this.#storage.conversation(id, context)); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#after(ALL_KEYS, () => this.#storage.scanConversations(query, limit, cursor, context)); + } + + entry(id: EntryId, context: Context): Promise; + entry(conversationId: ConversationId, id: EntryId, context: Context): Promise; + entry( + first: EntryId | ConversationId, + second: EntryId | Context, + third?: Context, + ): Promise { + if (third === undefined) { + const id = first as EntryId; + return this.#after(entryKey(id), () => this.#storage.entry(id, second as Context)); + } + const conversationId = first as ConversationId; + const id = second as EntryId; + // Visibility follows the conversation's immutable ancestry; the conversation may be unpublished itself. + return this.#after(conversationKey(conversationId), () => + this.#after(entryKey(id), () => this.#storage.entry(conversationId, id, third)), + ); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#afterEntries(conversationId, () => + this.#storage.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context), + ); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#afterEntries(query.conversationId, () => this.#storage.scanEntries(query, limit, cursor, context)); + } + + task(id: TaskId, context: Context): Promise | undefined> { + return this.#after(taskKey(id), () => this.#storage.task(id, context)); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise, Cursor>> { + return this.#after(ALL_KEYS, () => this.#storage.scanTasks(query, limit, cursor, context)); + } + + submission(id: SubmissionId, context: Context): Promise { + return this.#after(submissionKey(id), () => this.#storage.submission(id, context)); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#after(ALL_KEYS, () => this.#storage.scanSubmissions(query, limit, cursor, context)); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + return this.#after(requestKey(conversationId, requestId), () => + this.#storage.submissionByRequest(conversationId, requestId, context), + ); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#after(documentKey(addressId(address)), () => this.#storage.findDocument(address, at, context)); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + return this.#after(incarnationKey(id), () => this.#storage.document(id, at, context)); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#after(scopeKey(query.scope), () => this.#storage.scanDocuments(query, limit, cursor, context)); + } + + close(context: Context): Promise { + return this.#storage.close(context); + } + + /** + * Run a read of a conversation's entries once neither its appends nor its creation has an unpublished writer: Storage + * refuses the read of a conversation it does not have yet. + */ + #afterEntries(conversationId: ConversationId, read: () => Promise): Promise { + return this.#after(conversationKey(conversationId), () => this.#after(appendKey(conversationId), read)); + } + + /** Run `read` once `key` has no unpublished writer. */ + #after(key: string, read: () => Promise): Promise { + const gate = this.#touch(key); + return gate === undefined ? read() : gate.then(read); + } +} diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 5f5df8778..135855515 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -20,6 +20,7 @@ import type { CommitOptions, CommitPersistence, CommitPublication, + CommitResult, ConversationDocFamilyToken, ConversationDocToken, ConversationId, @@ -63,6 +64,9 @@ import type { TaskRecord, Tx, } from "../types.ts"; +import { CachedStorage, scopeOf } from "./cached-storage.ts"; +import { ConversationTails, type HeadMarker } from "./conversation-tails.ts"; +import { ALL_KEYS, appendKey, conversationKey, documentKey, GatedStorage, incarnationKey } from "./keys.ts"; import { type AdoptedDocuments, type HeldDocument, @@ -72,6 +76,7 @@ import { type TransactionHost, type TransactionScope, } from "./transaction.ts"; +import { type PublishFailure, Writer } from "./writer.ts"; /** * What the Session's own components observe of one commit (`followCommits()`): a stored commit with its own changes, or @@ -96,6 +101,27 @@ const NO_HOLDS: readonly Hold[] = []; const NO_OPERATIONS: readonly Op[] = []; +/** A line job's result: its value, delivered once `after`, the publication of its commit, settles, if it has one. */ +type Ran = { readonly value: T; readonly after?: Promise }; + +/** A job on the line: its index in line order, and whether a touch of it waited for an unpublished writer. */ +type Job = { + readonly index: number; + hit: boolean; + /** Called once its outcome settles: its commit published, or never to be (`onJobEnd()`). */ + ends?: (() => void)[]; +}; + +const NO_KEYS: readonly string[] = []; + +/** + * Jobs admitted to the line whose outcome is not delivered yet, beyond which the line waits. A count, not bytes: what + * waits is bounded by what one commit can hold (tool output by its output limits), times the count. + */ +const MAX_OUTSTANDING = 256; + +const ignore = (): void => {}; + /** * Queues a commit's host dispatch as one microtask through `then`: in the same queue as `queueMicrotask()`, which Node * wraps in an async resource that costs about twice the memory and three times the time per stored commit. @@ -172,10 +198,16 @@ export function persistWithin(ms: number | undefined): number { * Only committed state is observable. A deferred commit (`CommitPersistence`) is committed in memory first: it is * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, * for the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, - * which store them. Every commit callback, preparation, Storage settlement, and adoption runs while the line is held. A - * stored commit is published when Storage adopts it: inside `adopt` its telemetry events and the Session's own - * observers, one microtask later `observeCommits()` observers, before every wait the commit resolves and every abort - * it queues, all still on the line. Host listeners and watch listeners have no order. + * which store them. + * + * Group commit: the line runs one job at a time, a commit's callback and settlement; a commit with writes then joins + * the writer's pending batch (logical admission, `Writer`) and the line goes on. The writer stores every commit admitted + * while no call is in flight in one `commit()` call, off the line, and publishes them one by one, in line order, a + * macrotask apart: the read caches, adoption, telemetry events, the Session's own observers, and one microtask later + * `observeCommits()` observers, before every wait the commit resolves and every abort it queues. Host listeners and + * watch listeners have no order. A deferred commit is published when the commits before it are. A job's every read + * touches its key (`touch()`); while an unpublished commit writes that key, the job waits for its publication, so no + * job observes state that is not stored. Outcomes are delivered in line order (`#enqueue()`). * * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other * than for a cancelled caller, fails the Session, as does a commit it cannot adopt. That call gets the error; every later @@ -184,7 +216,14 @@ export function persistWithin(ms: number | undefined): number { * Storage's to retry. Reopening recovers from what was committed. */ export class SessionImpl implements Session { - readonly #storage: Storage; + readonly #storage: GuardedStorage; + /** Newest entry and head marker per conversation; none for a read-only Session, whose Storage another process writes. */ + readonly #tails: ConversationTails | undefined; + /** + * The current documents of each live task this Session created, so the commit that ends a task retires them without + * a scan. An entry starts with its task's creation and goes when the task is terminal: bounded by live tasks. + */ + readonly #taskDocuments = new Map>(); readonly #documents = new Map(); /** Cached documents with no observer attached, by address, oldest use first, with the time of that use. */ readonly #idleDocuments = new Map(); @@ -210,7 +249,27 @@ export class SessionImpl implements Session { readonly #internalListeners = new Set<(publication: ObservedCommit, context: Context) => void>(); readonly #closeListeners = new Set<() => void>(); readonly #host: TransactionHost; + /** The record cache under the guard, which the writer applies each batch to at its publication. */ + readonly #cached: CachedStorage | undefined; + readonly #writer: Writer; + /** The Storage jobs on the line read through: each read waits while its key has an unpublished writer. */ + readonly #gated: GatedStorage; + /** The end of the last line job: the line runs one job at a time. */ #tail: Promise = Promise.resolve(); + /** Line index of the next job; outcomes are delivered in this order. */ + #nextIndex = 0; + /** The job on the line, while one runs: its index, and whether it waited at a gate. */ + #current: Job | undefined; + /** Settles once every job so far has its outcome delivered. */ + #fence: Promise = Promise.resolve(); + #lastDelivered = -1; + /** Jobs that started on the line and have no outcome delivered yet (`MAX_OUTSTANDING`). */ + #outstanding = 0; + #room: PromiseWithResolvers | undefined; + /** The line index whose outcome the failure is: every later job's outcome is `SessionFailed`, except `#ownOutcomes`. */ + #failAt: number | undefined; + /** Later jobs whose outcome is their own error all the same: the other parts of a storage call that rejected. */ + readonly #ownOutcomes = new Set(); #closing: Promise | undefined; #failure: { readonly error: unknown } | undefined; /** Set when close reaches the backend: calls that fail from then on fail because it closes. */ @@ -242,7 +301,22 @@ export class SessionImpl implements Session { // Waits race it; none has to. this.#failed.promise.catch(() => {}); this.#readOnly = readOnly; - this.#storage = new GuardedStorage(storage, this); + // A read-only Session shows a Storage another process writes, so nothing it reads stays current. + this.#cached = readOnly ? undefined : new CachedStorage(storage); + this.#storage = new GuardedStorage(this.#cached ?? storage, this); + this.#gated = new GatedStorage(this.#storage, (key) => this.touch(key)); + this.#writer = new Writer({ + commit: (batches, receive, context) => this.#storage.commit(batches, receive, context), + stored: (writes) => { + this.#cached?.published(writes); + this.#tails?.apply(writes); + }, + publishing: (publish, context) => this.#publishing(publish, context), + failure: () => this.#failure, + fail: (error, at, own) => this.fail(error, undefined, at, own), + report: (error, context) => this.report(error, LISTENER, context), + }); + this.#tails = readOnly ? undefined : new ConversationTails(); this.#timers = hostTimers(setTimer, (error) => this.report(error, { kind: "host_error", callback: "setTimer" })); // Cache bookkeeping must never change the outcome of the operation it follows: a clock or a retention setting // that throws counts as now, and as the default. @@ -268,7 +342,8 @@ export class SessionImpl implements Session { } }; this.#host = { - storage: this.#storage, + storage: this.#gated, + touch: (key) => this.touch(key), now, cached: (id) => { const cached = this.#documents.get(id); @@ -284,6 +359,9 @@ export class SessionImpl implements Session { if (this.#documents.get(id)?.record.id === recordId) this.#dropDocument(id); }, conversationCreated: (tx, record) => this.conversationCreated(tx, record), + taskDocuments: (taskId) => this.#taskDocuments.get(taskId)?.values(), + latestEntryId: (conversationId, context) => this.latestEntryId(conversationId, context), + latestHeadMarker: (conversationId, context) => this.latestHeadMarker(conversationId, undefined, context), }; } @@ -292,6 +370,69 @@ export class SessionImpl implements Session { return this.#storage; } + /** + * Internal, on the Session line: the newest entry ID visible from `conversationId`. Known from the Session's own + * commits when it owns its Storage, so it costs no Storage read after the first. + */ + latestEntryId(conversationId: ConversationId, context: Context): Promise { + const gate = this.#touchEntries(conversationId); + const latest = (): Promise => { + const read = async () => + (await this.#storage.scanEntries({ conversationId }, 1, undefined, context)).items[0]?.id; + return this.#tails === undefined ? read() : this.#tails.latest(conversationId, read); + }; + return gate === undefined ? latest() : gate.then(latest); + } + + /** + * Internal: the newest entry ID of `conversationId` as known from the Session's commits right now, without reading; + * `undefined` when not known (and always for a read-only Session). + */ + knownLatestEntryId(conversationId: ConversationId): EntryId | null | undefined { + return this.#tails?.peek(conversationId); + } + + /** + * Internal, on the Session line: the newest visible head marker of `conversationId` at or before `atOrBefore`, any + * when `undefined`; known from the Session's own commits as `latestEntryId()` is. + */ + latestHeadMarker( + conversationId: ConversationId, + atOrBefore: EntryId | undefined, + context: Context, + ): Promise { + const gate = this.#touchEntries(conversationId); + const marker = (): Promise => { + const read = () => this.#storage.findLatestHeadMarker(conversationId, atOrBefore, context); + return this.#tails === undefined ? read() : this.#tails.marker(conversationId, atOrBefore, read); + }; + return gate === undefined ? marker() : gate.then(marker); + } + + /** + * On the line: wait for what an unpublished commit wrote of a conversation's entries, its appends and its creation, + * as reading them from Storage fails for a conversation not stored yet. + */ + #touchEntries(conversationId: ConversationId): Promise | undefined { + const created = this.touch(conversationKey(conversationId)); + const appended = this.touch(appendKey(conversationId)); + if (created === undefined) return appended; + return appended === undefined ? created : Promise.all([created, appended]).then(ignore); + } + + /** + * Internal, on the line: wait, while `key` has an unpublished writer (for `ALL_KEYS`, while anything is unpublished), + * until it is published, then check that the Session is healthy; `undefined` when there is nothing to wait for. A + * job's every read touches its key first, so it never observes state an earlier job admitted and the writer has not + * published. + */ + touch(key: string): Promise | undefined { + const gate = key === ALL_KEYS ? this.#writer.drained() : this.#writer.gate(key); + if (gate === undefined) return undefined; + if (this.#current !== undefined) this.#current.hit = true; + return gate.then(() => this.#assertHealthy()); + } + commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise { return this.commitWith(change, context, undefined, options); } @@ -299,11 +440,15 @@ export class SessionImpl implements Session { /** * Internal: fail the Session with `error`, the first wins, and report it once. Admission ends, and close listeners run * now; they read the error from `failure`. The Storage guard calls this; the Harness also calls it for a throw in its - * scheduler's own commits. + * scheduler's own commits. `at` is the line index of the job whose outcome the failure is, by default the job on the + * line: later jobs' outcomes become `SessionFailed`. */ - fail(error: unknown, context?: Context): void { + fail(error: unknown, context?: Context, at?: number, own?: readonly number[]): void { if (this.#failure !== undefined) return; this.#failure = { error }; + // The job that caused it keeps its own outcome; without one, every outcome not delivered yet is `SessionFailed`. + this.#failAt = at ?? this.#current?.index ?? this.#lastDelivered; + if (own !== undefined) for (const index of own) this.#ownOutcomes.add(index); this.#failed.reject(new SessionFailed(error)); // Seal first, so a report handler that calls back finds the Session failed. Closing runs the close listeners // synchronously; a failing backend close is the caller's to see, and here nobody waits. @@ -329,7 +474,41 @@ export class SessionImpl implements Session { } catch (error) { return Promise.reject(error); } - return this.#enqueue(() => this.#runCommit(change, context, scope, withinMs)); + return this.#enqueue((index) => this.#runCommit(change, context, scope, withinMs, index)); + } + + /** + * Internal: `commitWith()` for a commit of the scheduler's own whose outcome goes only to scheduler code. When it + * writes nothing and waited at no gate, it is equivalent to having run before every unpublished commit, so its + * outcome is delivered at once, ahead of theirs. One that writes is never exempt: its outcome waits for its own + * publication and every earlier outcome, so no invocation starts on a reservation that is not durable. + */ + commitInternal(change: (tx: Transaction) => T | Promise, context: Context): Promise { + try { + this.assertUsable(); + } catch (error) { + return Promise.reject(error); + } + return this.#enqueue((index) => this.#runCommit(change, context, undefined, undefined, index), true); + } + + /** + * Internal, on the line: call `end` once the outcome of the job on the line settles, its commit published or never to + * be; at once off the line. + */ + onJobEnd(end: () => void): void { + const job = this.#current; + if (job === undefined) { + end(); + return; + } + if (job.ends === undefined) job.ends = [end]; + else job.ends.push(end); + } + + /** Internal: the Storage jobs on the line read through (`touch()`); off the line, use the plain one. */ + get lineStorage(): Storage { + return this.#gated; } /** How long a commit may stay in memory before it is stored; `undefined` when it is immediate. */ @@ -349,8 +528,12 @@ export class SessionImpl implements Session { return Transaction.ownedBy(tx, this.#host); } - /** Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. */ - readOnLine(job: () => Promise): Promise { + /** + * Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. It reads + * through `storage`, whose reads wait for the publication of what the line admitted before; its outcome comes after + * theirs. + */ + readOnLine(job: (storage: Storage) => Promise): Promise { try { this.assertUsable(); } catch (error) { @@ -358,7 +541,7 @@ export class SessionImpl implements Session { } return this.#enqueue(async () => { this.#assertHealthy(); - return job(); + return { value: await job(this.#gated) }; }); } @@ -420,7 +603,7 @@ export class SessionImpl implements Session { ? cached : await this.#enqueue(async () => { this.#assertHealthy(); - return this.#loadDocument(definition, resolved.id, resolved.address, context); + return { value: await this.#loadDocument(definition, resolved.id, resolved.address, context) }; }); if (loaded === undefined) return undefined; checkRecordScope(definition, loaded.record); @@ -574,7 +757,7 @@ export class SessionImpl implements Session { const context = args[resolved.nextArgument + 1] as Context; return this.#enqueue(async () => { this.#assertHealthy(); - const storedEntry = await this.#storage.entry(conversationId, at, context); + const storedEntry = await this.#gated.entry(conversationId, at, context); if (storedEntry === undefined) { throw new Error(`Entry ${at} is not visible from conversation ${conversationId}`); } @@ -582,13 +765,13 @@ export class SessionImpl implements Session { ...resolved.address, scope: { kind: "conversation", conversationId: storedEntry.entry.conversationId }, }; - const record = await this.#storage.findDocument(address, storedEntry.commitSeq, context); - if (record === undefined) return undefined; - const stored = await this.#storage.document(record.id, storedEntry.commitSeq, context); + const record = await this.#gated.findDocument(address, storedEntry.commitSeq, context); + if (record === undefined) return { value: undefined }; + const stored = await this.#gated.document(record.id, storedEntry.commitSeq, context); if (stored === undefined) { throw new Error(`Historical document ${record.id} (${record.kind}) cannot be read`); } - return materializeDocument(definition, stored); + return { value: materializeDocument(definition, stored) }; }); } @@ -599,27 +782,29 @@ export class SessionImpl implements Session { this.#closing = Promise.resolve() .then(() => this.beforeClose()) .then(() => - this.#enqueue(async () => { + this.#maintain(async () => { + await this.#drained(); // Last, after the writes at close (`writeAtClose()`), while the trackers are still here. A failed Session // writes nothing; a failing store fails it, and the backend still closes. let unstored: { readonly error: unknown } | undefined; // Only with something to store: otherwise the backend's close begins in this very job. if (this.#failure === undefined && this.#holds.size > 0) { - await this.#storeHolds([...this.#holds.values()], cleanup).catch((error: unknown) => { + await this.#storeHolds(() => [...this.#holds.values()], cleanup).catch((error: unknown) => { unstored = { error }; }); } const observers = [...this.#commitListeners]; this.#commitListeners.clear(); // After the last publication: the commits admitted before the seal have settled, held changes are stored. - const end: WatchEnd = - this.#failure === undefined ? ENDED : { kind: "failed", error: this.#failure.error }; + // Commit observers are synchronous observers, all of them. + const end = closeEnds(this.#failure).observer; for (const observer of observers) observer.end(end); this.#internalListeners.clear(); this.#documents.clear(); this.#idleDocuments.clear(); this.#attachedDocuments.clear(); this.#holds.clear(); + this.#taskDocuments.clear(); try { await this.#storage.close(cleanup); } catch (error) { @@ -675,13 +860,31 @@ export class SessionImpl implements Session { * writes nothing. */ protected writeAtClose(writes: () => readonly StorageWrite[]): Promise { - return this.#enqueue(async () => { + return this.#maintain(async (index) => { + await this.#drained(); if (this.#failure !== undefined) return; const batch = writes(); - if (batch.length > 0) await this.#storage.commit(batch, () => {}, BACKGROUND_CONTEXT); + if (batch.length === 0) return; + const context = BACKGROUND_CONTEXT; + await this.#writer.admit({ + index, + writes: batch, + keys: [], + context, + publish: () => undefined, + discard: ignore, + }).outcome; }); } + /** + * Keys a commit writes besides its records' (`Transaction.writeKeys()`), at its admission, on the line: a Harness + * adds the ownership memberships its task changes write. A plain Session adds none. + */ + protected membershipKeys(_tx: Transaction): readonly string[] | Promise { + return NO_KEYS; + } + /** Runs after close seals admission and before the line closes Storage; must not reject. */ protected beforeClose(): Promise { return Promise.resolve(); @@ -689,9 +892,10 @@ export class SessionImpl implements Session { /** * Attach a synchronous observer of every stored commit, with exactly what it stored (`CommitPublication`). Its - * `advance` runs one microtask after Storage adopted the commit, on the line, before every wait the commit resolves - * and every abort it queues (in no order with watch listeners); it must not block or call Session operations. A throw - * is reported, detaches it, and ends it `listener_error`; `end` gets the Session's close after its last publication. + * `advance` runs one microtask after the commit's publication, which follows its storage call, in line order, before + * every wait the commit resolves and every abort it queues (in no order with watch listeners); it must not block or + * call Session operations. A throw is reported, detaches it, and ends it `listener_error`; `end` gets the Session's + * close after its last publication. */ observeCommits(observer: CommitObserver): CommitObservation { this.assertUsable(); @@ -706,8 +910,8 @@ export class SessionImpl implements Session { /** * Internal: follow every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state - * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. For a stored - * commit they run inside Storage's `adopt`, so they call no Storage method, run no task code, and contain every host + * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. They run at + * the commit's publication, synchronously, so they call no Storage method, run no task code, and contain every host * callback; the waits they resolve and the aborts they queue run after host observers. */ followCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { @@ -728,8 +932,8 @@ export class SessionImpl implements Session { /** Drop every loaded tracker on the mutation line, held changes stored first; later access cold-loads from Storage. */ unloadDocuments(): Promise { - return this.#enqueue(async () => { - if (this.#failure === undefined) await this.#storeHolds([...this.#holds.values()], BACKGROUND_CONTEXT); + return this.#maintain(async () => { + if (this.#failure === undefined) await this.#storeHolds(() => [...this.#holds.values()], BACKGROUND_CONTEXT); this.#documents.clear(); this.#idleDocuments.clear(); }); @@ -737,7 +941,8 @@ export class SessionImpl implements Session { /** * Internal, for tests: how many documents the cache holds, how many are idle, how many incarnations are pinned, how - * many sweep jobs were queued on the line so far, and how many documents have held changes. + * many sweep jobs were queued on the line so far, how many documents have held changes, how many live tasks have + * their documents indexed, and how many conversations have their newest entry kept. */ get documentCacheSizes(): { readonly cached: number; @@ -745,6 +950,8 @@ export class SessionImpl implements Session { readonly attached: number; readonly sweeps: number; readonly held: number; + readonly taskDocuments: number; + readonly tails: number; } { return { cached: this.#documents.size, @@ -752,6 +959,20 @@ export class SessionImpl implements Session { attached: this.#attachedDocuments.size, sweeps: this.#sweepJobs, held: this.#holds.size, + taskDocuments: this.#taskDocuments.size, + tails: this.#tails?.size ?? 0, + }; + } + + /** + * Internal, for tests that bound memory: jobs whose outcome is not delivered yet, keys with an unpublished writer, + * and record reads in flight in the cache. All three are 0 once the line is idle. + */ + get lineSizes(): { readonly outstanding: number; readonly unpublished: number; readonly reads: number } { + return { + outstanding: this.#outstanding, + unpublished: this.#writer.unpublished, + reads: this.#cached?.readsInFlight ?? 0, }; } @@ -784,16 +1005,22 @@ export class SessionImpl implements Session { */ #scheduleDocumentSweep(): void { if (this.#closing !== undefined) return; - const oldest = this.#idleDocuments.values().next(); - if (oldest.done === true) return; - const at = oldest.value + this.#documentRetentionMs(); + // Documents with unpublished changes are not swept: their publication touches them again. + let oldest: number | undefined; + for (const [addressId, since] of this.#idleDocuments) { + if (this.#writer.gate(documentKey(addressId)) !== undefined) continue; + oldest = since; + break; + } + if (oldest === undefined) return; + const at = oldest + this.#documentRetentionMs(); if (this.#documentSweep !== undefined && this.#documentSweep.at <= at) return; this.#documentSweep?.disarm?.(); // The pending sweep stays set until its job runs, so touches while the line is busy queue no more. const sweep: { readonly at: number; disarm: (() => void) | undefined } = { at, disarm: undefined }; const run = (): void => { this.#sweepJobs++; - this.#enqueue(async () => { + this.#maintain(async () => { if (this.#documentSweep === sweep) this.#documentSweep = undefined; if (this.#closing === undefined) this.#sweepDocuments(); }).catch(() => {}); @@ -823,6 +1050,8 @@ export class SessionImpl implements Session { const expired = this.#clock() - this.#documentRetentionMs(); for (const [addressId, since] of this.#idleDocuments) { if (since > expired) break; + // A document with an unpublished change stays until its publication touches it. + if (this.#writer.gate(documentKey(addressId)) !== undefined) continue; this.#dropDocument(addressId); } this.#scheduleDocumentSweep(); @@ -851,7 +1080,8 @@ export class SessionImpl implements Session { context: Context, scope: TransactionScope | undefined, withinMs: number | undefined, - ): Promise { + index: number, + ): Promise> { this.#assertHealthy(); context.abortSignal?.throwIfAborted(); const tx = new Transaction(this.#host, context, scope); @@ -876,7 +1106,7 @@ export class SessionImpl implements Session { if (writes.length === 0) { tx.discard(); this.#sweepDocuments(); - return result; + return { value: result }; } if (this.#readOnly) { // Refused before Storage: the Session stays healthy, and the caller sees why. @@ -884,63 +1114,93 @@ export class SessionImpl implements Session { this.#sweepDocuments(); throw new ReadOnlySession("a commit with writes"); } + const discard = (): void => tx.discard(); if (withinMs !== undefined) { const held = tx.holdable(); if (held !== undefined) { - this.#hold(tx, held, withinMs, context); + // Adopted in memory when the commits before it are published, as a part of no storage call. + const part = this.#writer.admit({ + index, + writes: undefined, + keys: tx.writeKeys(), + context, + publish: () => this.#hold(tx, held, withinMs, context), + discard, + }); this.#sweepDocuments(); - return result; + return { value: result, after: part.outcome }; } } - // The last step before Storage: held changes leave their queue only once nothing but Storage can fail. - const taken = this.#holds.size === 0 ? NO_HOLDS : tx.takeHeld((id) => this.#holds.get(id)); - // Storage adopts the batch synchronously once it is durable, before its next commit, and the commit is published in - // the same step, so every view changes with memory. A throw here would reach Storage, so the callback keeps its own - // error, and the first error of an own observer. - let adoption: - | { readonly documents: AdoptedDocuments; readonly failure?: { readonly error: unknown } } - | { readonly error: unknown } - | undefined; - const adopt = (committed: Seq): void => { - let documents: AdoptedDocuments; + let membership = this.membershipKeys(tx); + if (!Array.isArray(membership)) { try { - documents = tx.adopt(committed); + membership = await membership; } catch (error) { - adoption = { error }; - return; + tx.discard(); + this.#sweepDocuments(); + throw error; } - const failure = this.#publish(committed, writes, documents, context); - adoption = failure === undefined ? { documents } : { documents, failure }; - }; - try { - // Once admitted, caller cancellation does not interrupt Storage settlement. - await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); - } catch (error) { - // The guard has failed the Session with the storage error. A rejection after adoption leaves nothing staged to - // discard, and the commit, durable, stays published; an own observer's error that would have failed the Session - // is still reported. - if (adoption === undefined) tx.discard(); - else if ("documents" in adoption && adoption.failure !== undefined) { - this.report(adoption.failure.error, LISTENER, context); - } - throw error; } - if (adoption === undefined) { - const error = new Error("Storage committed without adopting the batch"); - this.fail(error); - throw error; - } - if ("error" in adoption) { - // Storage already committed; a failed adoption leaves memory behind durable state. - this.fail(adoption.error); - throw adoption.error; - } - if (taken.length > 0) this.#release(taken); - for (const document of adoption.documents.stored) this.#touchDocument(addressId(document.record)); - // The commit is durable and published, so it stands; memory behind an own observer's state fails the Session. - if (adoption.failure !== undefined) this.fail(adoption.failure.error, context); + // Logical admission. Held changes leave their queue only here, once nothing but Storage can fail; a commit never + // stores a hold adopted after it in line order. Taken before the keys, so the documents it stores held changes of + // are its writes too: a later job that reads one waits for it instead of storing them again. + const taken = this.#holds.size === 0 ? NO_HOLDS : tx.takeHeld((id) => this.#holds.get(id)); + const keys = tx.writeKeys(); + for (const key of membership as readonly string[]) keys.push(key); + const part = this.#writer.admit({ + index, + writes, + keys, + context, + publish: (seq) => this.#adopt(tx, writes, taken, seq!, context), + discard, + }); this.#sweepDocuments(); - return result; + return { value: result, after: part.outcome }; + } + + /** + * Publish a stored commit (`Writer`), after the read caches took its writes: adopt it in memory and run its + * publication. A throw of the adoption leaves memory behind durable state; the writer fails the Session with it. + */ + #adopt( + tx: Transaction, + writes: readonly StorageWrite[], + taken: readonly Hold[], + seq: Seq, + context: Context, + ): PublishFailure | undefined { + let documents: AdoptedDocuments; + try { + documents = tx.adopt(seq); + } catch (error) { + return { kind: "adoption", error }; + } + this.#indexTaskDocuments(tx, writes, documents, seq); + const failure = this.#publish(seq, writes, documents, context); + if (taken.length > 0) this.#release(taken); + for (const document of documents.stored) this.#touchDocument(addressId(document.record)); + return failure === undefined ? undefined : { kind: "listener", error: failure.error }; + } + + /** + * At publication: keep `#taskDocuments` in step with a stored commit: the tasks it creates start with no documents, + * task documents it creates join their task, ones it retires leave, and tasks it ends leave the index. + */ + #indexTaskDocuments(tx: Transaction, writes: readonly StorageWrite[], documents: AdoptedDocuments, seq: Seq): void { + for (const id of tx.createdTaskIds()) this.#taskDocuments.set(id, new Map()); + if (this.#taskDocuments.size === 0) return; + for (const change of documents.stored) { + if (change.type !== "document" || change.record.scope.kind !== "task") continue; + const known = this.#taskDocuments.get(change.record.scope.taskId); + if (known === undefined) continue; + if (change.kind === "retire") known.delete(change.record.id); + else if (change.record.createdAt === seq) known.set(change.record.id, change.record); + } + for (const write of writes) { + if (write.type === "task" && write.value.state.status === "terminal") + this.#taskDocuments.delete(write.value.id); + } } /** @@ -952,8 +1212,14 @@ export class SessionImpl implements Session { held: NonNullable>, withinMs: number, context: Context, - ): void { - const documents = tx.adoptHeld(); + ): PublishFailure | undefined { + let documents: DocumentCommitChange[]; + try { + documents = tx.adoptHeld(); + } catch (error) { + // As a stored commit's: memory could not adopt it; the writer fails the Session. + return { kind: "adoption", error }; + } const deadline = this.#clock() + withinMs; for (const { loaded, ops, base } of held) { const id = loaded.record.id; @@ -969,50 +1235,53 @@ export class SessionImpl implements Session { } this.#armHoldTimer(); const publication: ObservedCommit = { kind: "memory", changes: documents }; + let failure: PublishFailure | undefined; for (const listener of [...this.#internalListeners]) { // Our own observers are synchronous. try { listener(publication, context); } catch (error) { - this.fail(error, context); + failure ??= { kind: "listener", error }; } } + return failure; } /** - * Store the held changes of `holds` in one storage transaction, on the line. Memory already has them, so only - * `observeCommits()` observers see it, published as Storage adopts it. A failure fails the Session, as every Storage - * failure does. + * Store the held changes `select` returns in one storage transaction, through the writer, from a job on the line. A + * barrier: they are selected once every commit admitted before is published. Memory already has them, so only + * `observeCommits()` observers see it, published as the writer publishes it. A failure fails the Session, as every + * Storage failure does. */ - async #storeHolds(holds: readonly Hold[], context: Context): Promise { + async #storeHolds(select: () => readonly Hold[], context: Context): Promise { + await this.#drained(); + const holds = select(); if (holds.length === 0) return; const writes: Extract[] = []; const changes: DocumentCommitChange[] = []; + const keys: string[] = []; for (const hold of holds) { const { record, tracker, valueVersion: version } = hold.loaded; const value = tracker.value; const stored = storeHeld(hold, NO_OPERATIONS, value, version); writes.push({ type: "document.change", id: record.id, content: stored.content }); + keys.push(documentKey(hold.loaded.addressId), incarnationKey(record.id)); // Held documents are conversation documents. const conversationId = record.scope.kind === "conversation" ? record.scope.conversationId : undefined; changes.push({ type: "document", kind: "change", record, conversationId, version, value, ops: stored.ops }); } - let adopted = false; - const adopt = (seq: Seq): void => { - adopted = true; + const publish = (seq: Seq | undefined): undefined => { for (const [index, hold] of holds.entries()) { if (writes[index]!.content.kind === "base") hold.loaded.deltasSinceBase = 0; else hold.loaded.deltasSinceBase++; } - if (this.#commitListeners.size > 0) this.#notifyHosts({ seq, changes }, context); + if (this.#commitListeners.size > 0) this.#notifyHosts({ seq: seq!, changes }, context); + this.#release(holds); + return undefined; }; - await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); - if (!adopted) { - const error = new Error("Storage committed without adopting the batch"); - this.fail(error); - throw error; - } - this.#release(holds); + // Run from a job on the line, which a failure it causes is the outcome of. + const index = this.#current!.index; + await this.#writer.admit({ index, writes, keys, context, publish, discard: ignore }).outcome; } /** After their store: forget `holds`, unpin their documents, and move the timer to the next deadline. */ @@ -1041,11 +1310,11 @@ export class SessionImpl implements Session { const timer: HoldTimer = { at, disarm: () => {}, fired: false }; const run = (): void => { timer.fired = true; - this.#enqueue(async () => { + this.#maintain(async () => { if (this.#holdTimer === timer) this.#holdTimer = undefined; // Close stores what is held itself. if (this.#closing !== undefined || this.#failure !== undefined) return; - await this.#storeHolds([...this.#holds.values()], BACKGROUND_CONTEXT); + await this.#storeHolds(() => [...this.#holds.values()], BACKGROUND_CONTEXT); }).catch(() => {}); }; this.#holdTimer = timer; @@ -1053,9 +1322,9 @@ export class SessionImpl implements Session { } /** - * Inside `adopt`: publish a stored commit. Its telemetry events first, then the dispatch to `observeCommits()` + * At publication: publish a stored commit. Its telemetry events first, then the dispatch to `observeCommits()` * observers, queued before the waits and aborts the observers wake, then the Session's own observers. Never throws: - * returns the first error of an own observer, which fails the Session once `Storage.commit()` returned. + * returns the first error of an own observer, which fails the Session once its storage call settled. */ #publish( seq: Seq, @@ -1115,7 +1384,7 @@ export class SessionImpl implements Session { return failure; } - /** Inside `adopt`: record a commit's telemetry event. A throwing adapter is reported, as `adopt` must not throw. */ + /** At publication: record a commit's telemetry event. A throwing adapter is reported, as publication must not throw. */ #event(name: string, attributes: SpanAttributes, context: Context): void { try { context.telemetry.addEvent(name, attributes); @@ -1125,10 +1394,9 @@ export class SessionImpl implements Session { } /** - * Inside `adopt`: call the `observeCommits()` observers with `publication` one microtask from now, outside the - * storage's synchronous step and, queued before the Session's own observers run, before the waits they resolve and - * the aborts they queue. - * The observers are those attached when it runs; their guards report a throw. + * At publication: call the `observeCommits()` observers with `publication` one microtask from now, and, queued + * before the Session's own observers run, before the waits they resolve and the aborts they queue. The observers are + * those attached when it runs; their guards report a throw. */ #notifyHosts(publication: CommitPublication, context: Context): void { void SETTLED.then(() => { @@ -1160,7 +1428,7 @@ export class SessionImpl implements Session { if (cancelled) throw cancellationError(signal!); const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); if (cancelled) throw cancellationError(signal!); - return loaded === undefined ? undefined : attach(loaded); + return { value: loaded === undefined ? undefined : attach(loaded) }; }); if (attached === undefined) return undefined; if (cancelled) { @@ -1269,7 +1537,7 @@ export class SessionImpl implements Session { }); unsubscribeClose = this.subscribeClose(() => { detach(); - observer.end(this.#failure === undefined ? ENDED : { kind: "failed", error: this.#failure.error }); + observer.end(closeEndOf(observer, closeEnds(this.#failure))); }); return observer; } @@ -1280,6 +1548,9 @@ export class SessionImpl implements Session { address: DocumentAddress, context: Context, ): Promise { + // What an unpublished commit wrote at the address is in neither the cache nor what Storage shows yet. + const gate = this.touch(documentKey(addressId)); + if (gate !== undefined) await gate; const cached = this.#documents.get(addressId); // A tracker serves only tokens of the version its value was materialized for; others reload from Storage. if (cached?.valueVersion === definition.version) { @@ -1287,9 +1558,12 @@ export class SessionImpl implements Session { return cached; } if (cached !== undefined) { - // The reload must find what memory holds: held changes are stored before their tracker goes. - const hold = this.#holds.get(cached.record.id); - if (hold !== undefined) await this.#storeHolds([hold], context); + // The reload must find what memory holds: held changes are stored before their tracker goes, after a barrier. + const id = cached.record.id; + await this.#storeHolds(() => { + const hold = this.#holds.get(id); + return hold === undefined ? NO_HOLDS : [hold]; + }, context); this.#dropDocument(addressId); } const record = await this.#storage.findDocument(address, "current", context); @@ -1310,13 +1584,150 @@ export class SessionImpl implements Session { return loaded; } - #enqueue(job: () => Promise): Promise { - const run = this.#tail.then(job); - this.#tail = run.then( - () => undefined, - () => undefined, + /** + * Run `job` on the line, after the jobs before it, and deliver its outcome in line order: once the jobs before it + * have theirs, and once its commit, if it admitted one, is published. With `exempt`, a job that admitted no commit and + * waited at no gate is delivered as soon as it has its outcome: it is equivalent to having run before every + * unpublished commit. A job + * whose outcome comes after the one a failure is gets `SessionFailed` instead. At most `MAX_OUTSTANDING` jobs wait + * for their delivery; beyond it the line waits. + */ + #enqueue(work: (index: number) => Promise>, exempt = false): Promise { + const job: Job = { index: this.#nextIndex++, hit: false }; + const ran = this.#tail.then(async () => { + while (this.#outstanding >= MAX_OUTSTANDING) { + this.#room ??= Promise.withResolvers(); + await this.#room.promise; + } + this.#outstanding++; + return this.#onLine(job, work); + }); + this.#tail = ran.then(ignore, ignore); + const outcome = ran.then( + (result) => { + if (result.after === undefined) { + this.#ended(job); + return result.value; + } + return result.after.then( + () => { + this.#ended(job); + return result.value; + }, + (error: unknown) => { + this.#ended(job); + throw error; + }, + ); + }, + (error: unknown) => { + this.#ended(job); + throw error; + }, ); - return run; + // Delivered later, after the outcomes before it: handled now, so an early rejection is not reported unhandled. + outcome.catch(ignore); + const fence = this.#fence; + this.#fence = fence.then(() => outcome).then(ignore, ignore); + const index = job.index; + const deliver = (fenced: boolean) => + (fenced ? fence.then(() => outcome) : outcome).then( + (value) => { + this.#delivered(index, fenced); + if (this.#poisoned(index)) throw new SessionFailed(this.#failure!.error); + return value; + }, + (error: unknown) => { + this.#delivered(index, fenced); + throw this.#poisoned(index) ? new SessionFailed(this.#failure!.error) : error; + }, + ); + // Only a job without writes is exempt: one with writes waits for its own publication and the outcomes before it. + return exempt + ? ran.then( + (result) => deliver(job.hit || result.after !== undefined), + () => deliver(job.hit), + ) + : deliver(true); + } + + /** Whether job `index` comes after the failure's boundary without an outcome of its own: it gets `SessionFailed`. */ + #poisoned(index: number): boolean { + return this.#failAt !== undefined && index > this.#failAt && !this.#ownOutcomes.has(index); + } + + /** The job's outcome settled: run what waited for it (`onJobEnd()`). */ + #ended(job: Job): void { + const ends = job.ends; + if (ends === undefined) return; + job.ends = undefined; + for (const end of ends) end(); + } + + /** Run a job of the Session's own upkeep on the line: no outcome to deliver, so no fence and no `SessionFailed`. */ + #maintain(work: (index: number) => Promise): Promise { + const job: Job = { index: this.#nextIndex++, hit: false }; + const ran = this.#tail.then(() => this.#onLine(job, work)); + this.#tail = ran.then(ignore, ignore); + return ran; + } + + /** Hold the line for job `index` until its line part, through admission, is done. Gates are its touches' own. */ + async #onLine(job: Job, work: (index: number) => Promise): Promise { + this.#current = job; + try { + return await work(job.index); + } finally { + this.#current = undefined; + } + } + + /** A `*` barrier: once every part admitted so far is published, or never will be. */ + async #drained(): Promise { + const drained = this.#writer.drained(); + if (drained !== undefined) await drained; + } + + /** + * Run one commit's publication inside the Storage's own scope (`Storage.publishing`), such as a store host's batch; + * directly without one. `undefined` when it ran at once, else settles once it ran. A Storage that throws instead of + * running it is reported, and the publication runs directly: the commit is stored, so it must be published. + */ + #publishing(publish: () => void, context: Context): Promise | undefined { + const publishing = this.#storage.publishing; + if (publishing === undefined) { + publish(); + return undefined; + } + const done = Promise.withResolvers(); + let ran = false; + const once = (): void => { + if (ran) return; + ran = true; + try { + publish(); + } finally { + done.resolve(); + } + }; + try { + publishing(once, context); + } catch (error) { + this.report(error, LISTENER, context); + once(); + } + return done.promise; + } + + /** Job `index` has its outcome: let the line go on past `MAX_OUTSTANDING`. */ + #delivered(index: number, fenced: boolean): void { + if (fenced) this.#lastDelivered = index; + this.#outstanding--; + const room = this.#room; + if (room !== undefined && this.#outstanding < MAX_OUTSTANDING) { + this.#room = undefined; + room.resolve(); + } } /** Internal: throw `SessionFailed` when failed, else an error when closing. */ @@ -1351,7 +1762,7 @@ type SyncObserver = { advance(call: A, context: Context): void; end(end: Watc export type GuardedObserver = SyncObserver & { detach(): void }; /** - * `observer`, a host's, behind the rule for observers that run inside Storage's `adopt`: one that throws, or whose + * `observer`, a host's, behind the rule for observers that run at a commit's publication: one that throws, or whose * `advance` returns a promise that rejects, is reported through `report`, detached, and ended `listener_error`, so the * throw never reaches `adopt`; nothing reaches it after its detach or end, and from then on the guard references * neither it nor its source. `detach` is the source's; the returned `detach()` is the observation's. @@ -1419,6 +1830,28 @@ function toError(error: unknown): Error { /** The end of an observer whose source closed. */ export const ENDED: WatchEnd = { kind: "ended" }; +/** + * How a Session's close ends what follows it (decision "Slice 8 S1"), made once per close: an app's watch `ended`; a + * synchronous observer (an adapter following the Session, which may open it again) `failed` with "session closed", so + * it follows the next Session; `ended` stays for an observed owner that ended (a task document's task). A failed + * Session ends both `failed` with its error. + */ +export function closeEnds(failure: { readonly error: unknown } | undefined): { + readonly watch: WatchEnd; + readonly observer: WatchEnd; +} { + if (failure !== undefined) { + const failed: WatchEnd = { kind: "failed", error: failure.error }; + return { watch: failed, observer: failed }; + } + return { watch: ENDED, observer: { kind: "failed", error: new Error("session closed") } }; +} + +/** The end `ends` give `follower`: a watch's, or an observer's. */ +export function closeEndOf(follower: unknown, ends: ReturnType): WatchEnd { + return follower instanceof Watch ? ends.watch : ends.observer; +} + /** * The `Observation` of a guarded observer attached at `value`. Built outside the attaching closure, so a detached * observation keeps nothing but `value` and the spent guard alive. @@ -1498,9 +1931,29 @@ class GuardedStorage implements Storage { this.#latch = latch; } - /** Never exempt, whatever it throws: once admitted, whether a failed batch committed is unknown. */ - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { - return this.#call(undefined, () => this.#storage.commit(writes, adopt, context)); + /** The backend's own scope for a publication (`Storage.publishing`), when it has one; not a call to guard. */ + get publishing(): Storage["publishing"] { + return scopeOf(this.#storage); + } + + /** + * Guarded like every call, but its outcome is the Session writer's to judge: it publishes the batches received first, + * then fails the Session for a refusal or a rejection, and for a Session that failed meanwhile. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + this.#assertHealthy(); + if (this.#closing) throw new Error("Session is closed"); + const call = (async () => this.#storage.commit(batches, receive, context))(); + this.#underway.add(call); + try { + return await call; + } finally { + this.#underway.delete(call); + } } mintId>(): Promise { diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index 857728fba..a25197a74 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -57,6 +57,7 @@ import type { } from "../types.ts"; import { ROOT_CONVERSATION_ID } from "../types.ts"; import { prepareForkDocumentCopies } from "./forks.ts"; +import { documentKey, incarnationKey, scopeKey, tableKeys } from "./keys.ts"; type AnyTaskRecord = TaskRecord; @@ -119,7 +120,10 @@ export type AdoptedDocuments = { /** Session services used by a transaction while it holds the mutation line. */ export interface TransactionHost { + /** Reads wait while their key has an unpublished writer (`GatedStorage`). */ readonly storage: Storage; + /** Wait while `key` has an unpublished writer; `undefined` when it has none. */ + touch(key: string): Promise | undefined; /** Wall clock for task lifecycle times. */ now(): number; /** Return the cached current incarnation without loading. */ @@ -137,6 +141,17 @@ export interface TransactionHost { evict(addressId: string, recordId: DocumentId): void; /** Stage writes that belong to every newly created or forked conversation, in its creating transaction. */ conversationCreated(tx: Transaction, record: ConversationRecord): Promise; + /** The newest entry ID visible from a conversation, known from the Session's commits. */ + latestEntryId(conversationId: ConversationId, context: Context): Promise; + /** + * The current documents of a live task when the Session knows all of them (it created the task), else `undefined`. + */ + taskDocuments(taskId: TaskId): Iterable | undefined; + /** The newest visible head marker of a conversation, known from the Session's commits. */ + latestHeadMarker( + conversationId: ConversationId, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>; } /** Committed and candidate state for one task touched by this transaction. */ @@ -304,9 +319,11 @@ export class Transaction implements Tx { } latestHeadMarker(conversationId: ConversationId) { - return this.#read("latestHeadMarker", () => - this.#host.storage.findLatestHeadMarker(conversationId, undefined, this.#context), - ); + return this.#read("latestHeadMarker", () => this.#host.latestHeadMarker(conversationId, this.#context)); + } + + latestEntryId(conversationId: ConversationId): Promise { + return this.#read("latestEntryId", () => this.#host.latestEntryId(conversationId, this.#context)); } scanTasks(query: TaskQuery, limit: number, cursor?: Cursor) { @@ -548,6 +565,13 @@ export class Transaction implements Tx { return records; } + /** Internal: IDs of the tasks this transaction creates. */ + createdTaskIds(): TaskId[] { + const ids: TaskId[] = []; + for (const [id, task] of this.#tasksById) if (task.write?.kind === "create") ids.push(id); + return ids; + } + /** Internal: conversations this transaction created or forked so far. */ stagedConversations(): ConversationRecord[] { const records: ConversationRecord[] = []; @@ -717,6 +741,8 @@ export class Transaction implements Tx { } async #findRetirement(entry: DocumentEntry): Promise { + const gate = this.#host.touch(documentKey(entry.addressId)); + if (gate !== undefined) await gate; const record = this.#host.cached(entry.addressId)?.record ?? (await this.#host.storage.findDocument(entry.address, "current", this.#context)); @@ -850,6 +876,22 @@ export class Transaction implements Tx { return taken; } + /** + * Internal: the keys this commit writes, once settled (and after `takeHeld()`), for the Session's unpublished write + * set: its table records, and the address and incarnation of every document it creates, changes, or retires, with + * the scope of every incarnation it creates or retires. + */ + writeKeys(): string[] { + const keys: string[] = []; + tableKeys(this.#writes, keys); + for (const plan of this.#plans) { + if (plan.content === undefined && !plan.retire) continue; + keys.push(documentKey(plan.addressId), incarnationKey(plan.record.id)); + if (plan.retire || !("createdAt" in plan.record)) keys.push(scopeKey(plan.record.scope)); + } + return keys; + } + /** Stage a content write for an incarnation, before its retirement when it retires. */ #insertBeforeRetirement(write: Extract): void { const at = this.#writes.findIndex( @@ -944,8 +986,21 @@ export class Transaction implements Tx { plan.retire = true; retiring.add(plan.record.id); } + const retire = (record: DocumentRecord): void => { + if (retiring.has(record.id)) return; + plans.push({ addressId: addressId(record), record, retire: true }); + retiring.add(record.id); + }; for (const taskId of terminalTaskIds) { if (this.#tasksById.get(taskId)?.write?.kind === "create") continue; + // The Session knows the documents of the tasks it created; any other task's are scanned. + const gate = this.#host.touch(scopeKey({ kind: "task", taskId })); + if (gate !== undefined) await gate; + const known = this.#host.taskDocuments(taskId); + if (known !== undefined) { + for (const record of known) retire(record); + continue; + } let cursor: Cursor | undefined; do { const page = await storage.scanDocuments( @@ -954,11 +1009,7 @@ export class Transaction implements Tx { cursor, this.#context, ); - for (const record of page.items) { - if (retiring.has(record.id)) continue; - plans.push({ addressId: addressId(record), record, retire: true }); - retiring.add(record.id); - } + for (const record of page.items) retire(record); cursor = page.next; } while (cursor !== undefined); } diff --git a/packages/durable/src/session/writer.ts b/packages/durable/src/session/writer.ts new file mode 100644 index 000000000..18fc73d3e --- /dev/null +++ b/packages/durable/src/session/writer.ts @@ -0,0 +1,361 @@ +// The Session's writer: commits the mutation line admitted, stored in batches off the line and published in line order. +import type { Context } from "@earendil-works/chord"; +import { withoutAbortSignal } from "@earendil-works/chord/context"; +import { SessionFailed } from "../errors.ts"; +import type { CommitResult, Seq, StorageWrite } from "../types.ts"; + +/** A part's own failure at publication: memory could not adopt it, or one of the Session's own observers threw. */ +export type PublishFailure = { readonly kind: "adoption" | "listener"; readonly error: unknown }; + +/** One logical commit the line admitted: stored (`writes`, never empty), or held in memory only (no `writes`). */ +export type PartSpec = { + /** Line index of the job that admitted it: a failure it causes makes only later jobs' outcomes `SessionFailed`. */ + readonly index: number; + readonly writes: readonly StorageWrite[] | undefined; + /** Keys it writes: a read of one waits for its publication. */ + readonly keys: readonly string[]; + readonly context: Context; + /** Publish it in memory, in line order, once it is durable (with its seq) or, held, once the parts before it are. */ + readonly publish: (seq: Seq | undefined) => PublishFailure | undefined; + /** It will never be published: abort what it prepared. */ + readonly discard: () => void; +}; + +/** An admitted part and its outcome. */ +export type Part = PartSpec & { + /** + * Resolves once it is published; rejects once it never will be, and also after publication when its storage call + * rejected or the Session failed meanwhile. Always handled, so a part nobody awaits rejects silently. + */ + readonly outcome: Promise; +}; + +type Admitted = Part & { + readonly settle: { resolve(): void; reject(error: unknown): void }; + /** Set once its outcome is settled. */ + done: boolean; + /** Opens once it is published or never will be; never rejects. */ + readonly gate: PromiseWithResolvers; +}; + +/** How a part that will not resolve settles: with an error of its own, which may be any value, or `SessionFailed`. */ +type FailedOutcome = { readonly kind: "own"; readonly error: unknown } | { readonly kind: "failed" }; + +const SESSION_FAILED: FailedOutcome = { kind: "failed" }; + +/** What the writer needs of its Session. */ +export type WriterHost = { + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise; + /** Apply a stored batch to the Session's read caches, right before it is published. */ + stored(writes: readonly StorageWrite[]): void; + /** + * Run one part's publication inside the Storage's own scope (`Storage.publishing`); `undefined` when it ran at once, + * else settles once it ran. + */ + publishing(publish: () => void, context: Context): Promise | undefined; + /** The Session's failure, if it failed. */ + failure(): { readonly error: unknown } | undefined; + /** + * Fail the Session; `at` is the line index of the first job whose outcome the failure is, and `own` the jobs after it + * whose outcome is still their own error (the other parts of a call that rejected), not `SessionFailed`. + */ + fail(error: unknown, at: number, own: readonly number[]): void; + /** Report an own observer's throw that cannot fail the Session, as it fails with a storage error. */ + report(error: unknown, context: Context): void; +}; + +/** + * The writer of one Session. The line admits each commit as a part once its callback and settlement succeeded; the + * writer sends every part admitted while no call is in flight as one `commit()` call, off the line, and publishes + * the parts of a call one by one, in line order, with a macrotask between two, so the continuations of the waits one + * resolves start before the next is published. Held commits are parts without writes, published when the parts before + * them are. + * + * Each part registers the keys it writes; until it is published, it is the newest unpublished writer of each, and a + * job that touches one waits for its gate. A key leaves when its newest writer is published, so the map holds one entry + * per key with an unpublished writer. + */ +export class Writer { + readonly #host: WriterHost; + /** Admitted parts not yet sent, in line order. */ + #queue: Admitted[] = []; + #flight = false; + /** Per key, the newest unpublished part that writes it. */ + readonly #unpublished = new Map(); + /** The newest admitted part that is not yet published, whose gate opens after every earlier one's. */ + #newest: Admitted | undefined; + + constructor(host: WriterHost) { + this.#host = host; + } + + /** Admit a part: register its keys, queue it, and send it at once unless a call is in flight. */ + admit(spec: PartSpec): Part { + const outcome = Promise.withResolvers(); + outcome.promise.catch(() => {}); + const part: Admitted = { + ...spec, + outcome: outcome.promise, + settle: { + resolve: () => { + part.done = true; + outcome.resolve(); + }, + reject: (error) => { + part.done = true; + outcome.reject(error); + }, + }, + done: false, + gate: Promise.withResolvers(), + }; + for (const key of spec.keys) this.#unpublished.set(key, part); + this.#newest = part; + this.#queue.push(part); + this.#pump(); + return part; + } + + /** The gate of the newest unpublished writer of `key`, or `undefined` when it has none. */ + gate(key: string): Promise | undefined { + return this.#unpublished.get(key)?.gate.promise; + } + + /** Opens once every part admitted so far is published or never will be. */ + drained(): Promise | undefined { + return this.#newest?.gate.promise; + } + + /** Keys with an unpublished writer, for tests that bound memory. */ + get unpublished(): number { + return this.#unpublished.size; + } + + #pump(): void { + if (this.#flight || this.#queue.length === 0) return; + this.#flight = true; + const parts = this.#queue; + this.#queue = []; + void this.#flush(parts); + } + + /** + * Send and publish `parts`, then what was admitted meanwhile. Never rejects: a throw the writer did not expect fails + * the Session and settles the parts (`#abandon()`), so no outcome or gate is stranded. + */ + async #flush(parts: readonly Admitted[]): Promise { + try { + await this.#send(parts); + } catch (error) { + this.#abandon(parts, error); + } + this.#flight = false; + this.#pump(); + } + + /** Store `parts` in one call, then publish them in order; settles every one of them. */ + async #send(parts: readonly Admitted[]): Promise { + const stored = parts.filter((part) => part.writes !== undefined); + const seqs: (Seq | undefined)[] = []; + let result: CommitResult | undefined; + let rejected: { readonly error: unknown } | undefined; + const failed = this.#host.failure(); + if (failed !== undefined) { + for (const part of parts) this.#unpublish(part, new SessionFailed(failed.error)); + return; + } + if (stored.length > 0) { + try { + result = await this.#host.commit( + stored.map((part) => part.writes!), + (index, seq) => { + seqs[index] = seq; + }, + withoutAbortSignal(stored[0]!.context), + ); + } catch (error) { + rejected = { error }; + } + } + // A call that settles once another failure failed the Session reports `SessionFailed` with that first error, + // whatever it returned (spec §10.1), and publishes nothing. + const failedMeanwhile = this.#host.failure(); + if (failedMeanwhile !== undefined) { + for (const part of parts) this.#unpublish(part, new SessionFailed(failedMeanwhile.error)); + return; + } + await this.#publish(parts, seqs, result, rejected); + } + + /** + * Publish the received parts and held ones in order, a macrotask apart, and settle every part (group commit §3.4). A + * refusal, a rejection, or a part whose adoption or own observer throws fails the Session once the parts before it + * are published and the host observers got the last one; the parts after it are not published, and reject with + * `SessionFailed`. What the failure decides (the outcomes, and the gates of the parts it concerns) is settled only + * once the Session failed, so no job waiting at such a gate, and no later outcome, goes on as if it had not. + */ + async #publish( + parts: readonly Admitted[], + seqs: readonly (Seq | undefined)[], + result: CommitResult | undefined, + rejected: { readonly error: unknown } | undefined, + ): Promise { + let stop: { readonly error: unknown; readonly at: number; readonly external?: true } | undefined; + /** + * Settled once the Session failed: each part, whether it was published, and its outcome: an error of its own (any + * value, `undefined` included), or `SessionFailed`. + */ + const failing: { readonly part: Admitted; readonly outcome: FailedOutcome; readonly published: boolean }[] = []; + let storedIndex = 0; + let published = 0; + for (const part of parts) { + const index = part.writes === undefined ? undefined : storedIndex++; + if (stop !== undefined) { + // A part after a rejected call's last receipt has the call's error; any other part after a stop was not sent, + // nor any part once something else failed the Session. + const outcome: FailedOutcome = + rejected !== undefined && index !== undefined && stop.external !== true + ? { kind: "own", error: rejected.error } + : SESSION_FAILED; + failing.push({ part, outcome, published: false }); + continue; + } + const seq = index === undefined ? undefined : seqs[index]; + if (index !== undefined && seq === undefined) { + const error = + rejected !== undefined + ? rejected.error + : index === result?.stored && "refused" in result + ? result.refused + : new Error("Storage committed without adopting the batch"); + failing.push({ part, outcome: { kind: "own", error }, published: false }); + stop = { error, at: part.index }; + continue; + } + // A macrotask between two parts: what the first resolves runs before the second is published. + if (published++ > 0) await hop(); + let failure: PublishFailure | undefined; + // Failed by something else meanwhile, during the hop or before a storage's scope runs the publication: nothing + // more of the call is published. Checked right before it would be, after every wait. + let external: { readonly error: unknown } | undefined; + const publication = this.#host.publishing(() => { + external = this.#host.failure(); + if (external !== undefined) return; + // Its keys leave first, so what its publication schedules sees it published; its waiters wake after it. + this.#unregister(part); + // A throw anywhere in it, the read caches included, leaves memory behind what is stored: like an adoption's. + try { + if (part.writes !== undefined) this.#host.stored(part.writes); + failure = part.publish(seq); + } catch (error) { + failure = { kind: "adoption", error }; + } + }, part.context); + if (publication !== undefined) await publication; + if (external !== undefined) { + stop = { error: external.error, at: part.index, external: true }; + failing.push({ part, outcome: SESSION_FAILED, published: false }); + continue; + } + if (failure?.kind === "adoption") { + failing.push({ part, outcome: { kind: "own", error: failure.error }, published: true }); + stop = { error: failure.error, at: part.index }; + continue; + } + // A part published while its call rejected is durable but rejects (spec §10.1); an own observer's throw is then + // reported, as the Session fails with the storage error. + if (rejected !== undefined) { + if (failure !== undefined) this.#host.report(failure.error, part.context); + failing.push({ part, outcome: { kind: "own", error: rejected.error }, published: true }); + continue; + } + // An own observer's throw: the commit stands; the Session fails after it. + if (failure !== undefined) stop = { error: failure.error, at: part.index }; + const failedMeanwhile = this.#host.failure(); + if (failedMeanwhile !== undefined && failure === undefined) { + part.settle.reject(new SessionFailed(failedMeanwhile.error)); + } else part.settle.resolve(); + this.#open(part); + } + if (rejected !== undefined || stop !== undefined) { + // After the host dispatch of the last part published, which is one microtask away. + await Promise.resolve(); + // Every part of the call that rejected keeps the storage error; the first of them is the failure's boundary. + const own = failing.flatMap(({ part, outcome }) => (outcome.kind === "own" ? [part.index] : [])); + if (rejected !== undefined) this.#host.fail(rejected.error, own[0]!, own); + else this.#host.fail(stop!.error, stop!.at, own); + } + for (const { part, outcome, published: stored } of failing) { + const error = outcome.kind === "own" ? outcome.error : this.#failed(stop!.error); + if (stored) { + part.settle.reject(error); + this.#open(part); + } else this.#unpublish(part, error); + } + // Admitted while this call ran, after the failure: never sent. + const failure = this.#host.failure(); + if (failure !== undefined) { + const queued = this.#queue; + this.#queue = []; + for (const part of queued) this.#unpublish(part, new SessionFailed(failure.error)); + } + } + + /** + * A throw the writer itself did not expect while sending or publishing `parts` (a bug): fail the Session at the first + * of them and settle every one not settled yet, so no outcome, gate, or close waits on it forever. + */ + #abandon(parts: readonly Admitted[], error: unknown): void { + this.#host.fail(error, parts[0]!.index, []); + for (const part of parts) { + if (!part.done) this.#unpublish(part, this.#failed(error)); + this.#unregister(part); + this.#open(part); + } + } + + /** `SessionFailed` with the Session's failure, or with `error` until the Session records it. */ + #failed(error: unknown): SessionFailed { + const failure = this.#host.failure(); + return new SessionFailed(failure !== undefined ? failure.error : error); + } + + /** Settle a part that will never be published: discard it, reject it with `error`, and open its gate. */ + #unpublish(part: Admitted, error: unknown): void { + part.discard(); + part.settle.reject(error); + this.#unregister(part); + this.#open(part); + } + + /** Drop the keys a part is still the newest writer of: it is published, or never will be. */ + #unregister(part: Admitted): void { + for (const key of part.keys) if (this.#unpublished.get(key) === part) this.#unpublished.delete(key); + } + + /** Open a part's gate: the jobs waiting for it go on. The newest part's opens `drained()` too. */ + #open(part: Admitted): void { + part.gate.resolve(); + if (this.#newest === part) this.#newest = undefined; + } +} + +/** + * Yield to the event loop once: `setImmediate` where there is one, otherwise a `MessageChannel` message (Workers, + * browsers). Microtasks queued before it, such as wait continuations, run first. + */ +function hop(): Promise { + if (typeof setImmediate === "function") return new Promise((resolve) => setImmediate(resolve)); + const channel = new MessageChannel(); + return new Promise((resolve) => { + channel.port1.onmessage = () => { + channel.port1.close(); + resolve(); + }; + channel.port2.postMessage(undefined); + }); +} diff --git a/packages/durable/src/storage/indexeddb.ts b/packages/durable/src/storage/indexeddb.ts index 8dbc255bb..8c6c4706e 100644 --- a/packages/durable/src/storage/indexeddb.ts +++ b/packages/durable/src/storage/indexeddb.ts @@ -5,6 +5,7 @@ import { apply, type Op } from "@earendil-works/chord/delta"; import { StorageRequestError } from "../errors.ts"; import { idFromNumber, seqFromNumber } from "../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, @@ -138,12 +139,33 @@ type IndexedDbStorageOptions = { readonly indexedDB?: IDBFactory; }; +/** A failure IndexedDB reported without an error of its own: the database's, like every `DOMException` it throws. */ +class IndexedDbFailure extends Error {} + +/** + * Whether `error`, or an error it was caused by (a copy's rejection wraps its cause), is IndexedDB's own: a + * `DOMException`, or a failure it reported without one. Any other error of a batch's checks is one of Durable's. + */ +function fromIndexedDb(error: unknown): boolean { + const seen = new Set(); + for (let at: unknown = error; at !== null && typeof at === "object" && !seen.has(at); ) { + if (at instanceof DOMException || at instanceof IndexedDbFailure) return true; + seen.add(at); + at = (at as { cause?: unknown }).cause; + } + return false; +} + const requestResult = (request: IDBRequest): Promise => new Promise((resolve, reject) => { request.addEventListener("success", () => resolve(request.result), { once: true }); - request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB request failed")), { - once: true, - }); + request.addEventListener( + "error", + () => reject(request.error ?? new IndexedDbFailure("IndexedDB request failed")), + { + once: true, + }, + ); }); const transactionResult = (transaction: IDBTransaction): Promise => @@ -151,12 +173,12 @@ const transactionResult = (transaction: IDBTransaction): Promise => transaction.addEventListener("complete", () => resolve(), { once: true }); transaction.addEventListener( "abort", - () => reject(transaction.error ?? new Error("IndexedDB transaction was aborted")), + () => reject(transaction.error ?? new IndexedDbFailure("IndexedDB transaction was aborted")), { once: true }, ); transaction.addEventListener( "error", - () => reject(transaction.error ?? new Error("IndexedDB transaction failed")), + () => reject(transaction.error ?? new IndexedDbFailure("IndexedDB transaction failed")), { once: true }, ); }); @@ -176,9 +198,13 @@ const cursorValues = ( new Promise((resolve, reject) => { const values: T[] = []; const request = source.openCursor(range, direction); - request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB cursor failed")), { - once: true, - }); + request.addEventListener( + "error", + () => reject(request.error ?? new IndexedDbFailure("IndexedDB cursor failed")), + { + once: true, + }, + ); request.addEventListener("success", () => { const cursor = request.result; if (cursor === null) { @@ -203,9 +229,13 @@ const firstCursorValue = ( ): Promise => new Promise((resolve, reject) => { const request = source.openCursor(range, direction); - request.addEventListener("error", () => reject(request.error ?? new Error("IndexedDB cursor failed")), { - once: true, - }); + request.addEventListener( + "error", + () => reject(request.error ?? new IndexedDbFailure("IndexedDB cursor failed")), + { + once: true, + }, + ); request.addEventListener("success", () => { const cursor = request.result; if (cursor === null) { @@ -417,29 +447,54 @@ export class IndexedDbStorage implements Storage { await requestResult(factory.deleteDatabase(name)); } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, _context: Context): Promise { + /** + * Commit `batches` in one IndexedDB transaction, each with its own seq. Each batch does every read and check, copy + * resolution included, against the state the earlier ones left before it issues its first write; the first one + * refused ends the call, and the transaction commits the ones before it. An error of IndexedDB itself aborts the + * transaction: the call rejects with nothing stored. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + _context: Context, + ): Promise { this.assertOpen(); - const documentActions = this.prepareDocumentActions(writes); - const seq = await this.transaction(ALL_STORES, "readwrite", async (transaction) => { + const seqs: Seq[] = []; + let refused: { readonly error: unknown } | undefined; + await this.transaction(ALL_STORES, "readwrite", async (transaction) => { const metaStore = transaction.objectStore(STORES.meta); const metadata = await requestResult(metaStore.get(META_KEY) as IDBRequest); if (metadata === undefined) throw new Error("Durable IndexedDB metadata is missing"); - const seq = seqFromNumber(metadata.nextSeq); - await this.checkGlobalIds(transaction, writes); - await this.checkDocumentActions(transaction, documentActions); - for (const write of writes) await this.applyTableWrite(transaction, write, seq); - await this.applyDocumentActions(transaction, documentActions, seq); + let next = metadata.nextSeq; let nextId = metadata.nextId; - for (const write of writes) { - const id = writeId(write); - if (id !== undefined) nextId = Math.max(nextId, id + 1); + for (const writes of batches) { + let documentActions: Map; + try { + documentActions = this.prepareDocumentActions(writes); + await this.checkGlobalIds(transaction, writes); + await this.checkDocumentActions(transaction, documentActions); + await this.resolveCopies(transaction, documentActions); + } catch (error) { + if (fromIndexedDb(error)) throw error; + refused = { error }; + break; + } + const seq = seqFromNumber(next++); + for (const write of writes) await this.applyTableWrite(transaction, write, seq); + await this.applyDocumentActions(transaction, documentActions, seq); + for (const write of writes) { + const id = writeId(write); + if (id !== undefined) nextId = Math.max(nextId, id + 1); + } + seqs.push(seq); + } + if (seqs.length > 0) { + await requestResult(metaStore.put({ key: META_KEY, nextId, nextSeq: next } satisfies MetaRow)); } - await requestResult(metaStore.put({ key: META_KEY, nextId, nextSeq: seq + 1 } satisfies MetaRow)); - return seq; }); - // The transaction completed: the batch is durable. - adopt(seq); - return seq; + // The transaction completed: the stored batches are durable. + for (const [index, seq] of seqs.entries()) receive(index, seq); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; } async mintId>(): Promise { @@ -1124,6 +1179,31 @@ export class IndexedDbStorage implements Storage { } } + /** Resolve each copy to a base of its source's state before this batch, which may not change the source. */ + private async resolveCopies(transaction: IDBTransaction, actions: Map): Promise { + for (const [id, action] of actions) { + if (action.copy === undefined) continue; + try { + const stored = await this.materializeDocument(transaction, action.copy.id, action.copy.at); + if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); + } + action.content = { kind: "base", version: stored.version, value: stored.value }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + private async applyDocumentActions( transaction: IDBTransaction, actions: ReadonlyMap, @@ -1132,27 +1212,8 @@ export class IndexedDbStorage implements Storage { const documents = transaction.objectStore(STORES.documents); const revisions = transaction.objectStore(STORES.documentRevisions); for (const [id, action] of actions) { - let content = action.content; - if (action.copy !== undefined) { - try { - const stored = await this.materializeDocument(transaction, action.copy.id, action.copy.at); - if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); - const create = action.create!; - if ( - stored.record.scope.kind !== "conversation" || - create.scope.kind !== "conversation" || - stored.record.kind !== create.kind || - stored.record.key !== create.key || - stored.record.history !== create.history || - stored.record.fork !== create.fork - ) { - throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); - } - content = { kind: "base", version: stored.version, value: stored.value }; - } catch (error) { - throw new Error(`Document copy ${id} was rejected`, { cause: error }); - } - } + // Copies are resolved to their content by `resolveCopies()`. + const content = action.content; let record: DocumentRecord; if (action.create !== undefined) { record = { diff --git a/packages/durable/src/storage/jsonl/storage.ts b/packages/durable/src/storage/jsonl/storage.ts index 56b219e00..6e8bcbf0e 100644 --- a/packages/durable/src/storage/jsonl/storage.ts +++ b/packages/durable/src/storage/jsonl/storage.ts @@ -3,6 +3,7 @@ import type { FileError, FileSystem } from "../../env/index.ts"; import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, Cursor, @@ -249,6 +250,8 @@ export class JsonlStorage implements Storage { private readonly liveTaskSidecars = new Set(); private closed = false; private poisonError: JsonlStoragePoisonedError | undefined; + /** Settles once the batch being appended is; its batches are in memory already (`commit()`). */ + private writing: Promise | undefined; private constructor(fs: FileSystem, directory: string, mainPath: string, options: JsonlStorageOptions) { this.fs = fs; @@ -275,19 +278,74 @@ export class JsonlStorage implements Storage { return storage; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { + /** + * Commit `batches` with one append per affected file: every batch's sidecar records first, then their markers in one + * write of `main.jsonl`. Each batch is checked against the state the earlier ones left, so memory applies each one + * as it passes, and reads wait until the appends are done; the first batch refused ends the batch. A torn append of + * the markers recovers a shorter complete prefix; any failure once a batch applied poisons the storage. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { this.assertUsable(); - const prepared = this.memory.prepareCommit(writes); - const encoded = this.encodeCommit(prepared.seq, prepared.writes); - const reclamations = this.planReclamations(prepared.writes, encoded); + const seqs: Seq[] = []; + const encoded: EncodedCommit[] = []; + const reclamations = new Map(); + let refused: { readonly error: unknown } | undefined; + const writing = Promise.withResolvers(); + this.writing = writing.promise; + try { + for (const writes of batches) { + let prepared: ReturnType; + try { + prepared = this.memory.prepareCommit(writes); + } catch (error) { + refused = { error }; + break; + } + const commit = this.encodeCommit(prepared.seq, prepared.writes); + // A sidecar an earlier batch replaces keeps what the later ones append, unless they replace it again. + for (const [file, content] of commit.sidecars) { + const replacement = reclamations.get(file); + if (replacement !== undefined) reclamations.set(file, replacement + content); + } + for (const [file, content] of this.planReclamations(prepared.writes, commit)) + reclamations.set(file, content); + seqs.push(prepared.apply()); + this.adoptSidecarState(prepared.writes); + encoded.push(commit); + } + if (encoded.length > 0) await this.append(encoded, context); + } catch (error) { + // Memory runs ahead of the file: once a batch applied, any failure poisons the storage before the reads + // waiting on the append go on, so none of them returns what is not stored. + if (seqs.length === 0) throw error; + throw this.poison(error instanceof Error ? error : new Error("JSONL commit failed", { cause: error })); + } finally { + this.writing = undefined; + writing.resolve(); + } + // The markers are appended: the batches are durable. Reclaiming sidecars comes after the receipts. + for (const [index, seq] of seqs.entries()) receive(index, seq); + await this.reclaimSidecars(reclamations, context); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; + } + + /** Append the sidecar records of `commits`, flushed with `fsync`, then their markers in one write of `main.jsonl`. */ + private async append(commits: readonly EncodedCommit[], context: Context): Promise { + const contents = new Map(); + for (const commit of commits) { + for (const [file, content] of commit.sidecars) contents.set(file, (contents.get(file) ?? "") + content); + } const sidecars = await Promise.all( - [...encoded.sidecars].map(async ([file, content]) => ({ + [...contents].map(async ([file, content]) => ({ file, content, path: await this.resolveFile(file, context), })), ); - for (const sidecar of sidecars) { const result = await this.fs.appendFile(sidecar.path, sidecar.content, context); if (!result.ok) throw this.poison(errorFromFile(`append to ${sidecar.file}`, result.error)); @@ -298,14 +356,9 @@ export class JsonlStorage implements Storage { if (!result.ok) throw this.poison(errorFromFile(`flush of ${sidecar.file}`, result.error)); } } - const marker = await this.fs.appendFile(this.mainPath, encoded.marker, context); + const markers = commits.length === 1 ? commits[0]!.marker : commits.map((commit) => commit.marker).join(""); + const marker = await this.fs.appendFile(this.mainPath, markers, context); if (!marker.ok) throw this.poison(errorFromFile(`append to ${MAIN_FILE}`, marker.error)); - const seq = prepared.apply(); - this.adoptSidecarState(prepared.writes); - // The marker is appended: the batch is durable. Reclaiming sidecars comes after adoption. - adopt(seq); - await this.reclaimSidecars(reclamations, context); - return seq; } async mintId>(): Promise { @@ -313,20 +366,20 @@ export class JsonlStorage implements Storage { } async conversation(id: ConversationId, context: Context) { - return this.store.conversation(id, context); + return (await this.readable()).conversation(id, context); } async scanConversations(query: ConversationQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanConversations(query, limit, cursor, context); + return (await this.readable()).scanConversations(query, limit, cursor, context); } entry(id: EntryId, context: Context): ReturnType; entry(conversationId: ConversationId, id: EntryId, context: Context): ReturnType; async entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, context?: Context) { if (context === undefined) - return this.store.entry(idFromNumber(idOrConversationId), idOrContext as Context); + return (await this.readable()).entry(idFromNumber(idOrConversationId), idOrContext as Context); if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID"); - return this.store.entry( + return (await this.readable()).entry( idFromNumber(idOrConversationId), idFromNumber(idOrContext), context, @@ -338,43 +391,43 @@ export class JsonlStorage implements Storage { atOrBeforeEntryId: EntryId | undefined, context: Context, ) { - return this.store.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context); + return (await this.readable()).findLatestHeadMarker(conversationId, atOrBeforeEntryId, context); } async scanEntries(query: EntryQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanEntries(query, limit, cursor, context); + return (await this.readable()).scanEntries(query, limit, cursor, context); } async task(id: TaskId, context: Context) { - return this.store.task(id, context); + return (await this.readable()).task(id, context); } async scanTasks(query: TaskQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanTasks(query, limit, cursor, context); + return (await this.readable()).scanTasks(query, limit, cursor, context); } async submission(id: SubmissionId, context: Context) { - return this.store.submission(id, context); + return (await this.readable()).submission(id, context); } async scanSubmissions(query: SubmissionQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanSubmissions(query, limit, cursor, context); + return (await this.readable()).scanSubmissions(query, limit, cursor, context); } async submissionByRequest(conversationId: ConversationId, requestId: string, context: Context) { - return this.store.submissionByRequest(conversationId, requestId, context); + return (await this.readable()).submissionByRequest(conversationId, requestId, context); } async findDocument(address: DocumentAddress, at: DocumentPoint, context: Context) { - return this.store.findDocument(address, at, context); + return (await this.readable()).findDocument(address, at, context); } async document(id: DocumentId, at: DocumentPoint, context: Context) { - return this.store.document(id, at, context); + return (await this.readable()).document(id, at, context); } async scanDocuments(query: DocumentQuery, limit: number, cursor: Cursor | undefined, context: Context) { - return this.store.scanDocuments(query, limit, cursor, context); + return (await this.readable()).scanDocuments(query, limit, cursor, context); } async close(context: Context): Promise { @@ -836,6 +889,12 @@ export class JsonlStorage implements Storage { return this.memory; } + /** The state to read once no batch is being appended: memory runs ahead of the file while one is. */ + private async readable(): Promise { + while (this.writing !== undefined) await this.writing; + return this.store; + } + private poison(cause: Error): JsonlStoragePoisonedError { this.poisonError ??= new JsonlStoragePoisonedError(cause); return this.poisonError; diff --git a/packages/durable/src/storage/memory.ts b/packages/durable/src/storage/memory.ts index dd72b2e3a..defcc0f56 100644 --- a/packages/durable/src/storage/memory.ts +++ b/packages/durable/src/storage/memory.ts @@ -3,6 +3,7 @@ import { applyImmutableBatches, type Op } from "@earendil-works/chord/delta"; import { StorageRequestError } from "../errors.ts"; import { idFromNumber, seqFromNumber } from "../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, @@ -260,10 +261,35 @@ export class MemoryStorage implements Storage { private nextSeq = 1; private closed = false; - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, _context: Context): Promise { - const seq = this.prepareCommit(writes).apply(); - adopt(seq); - return seq; + /** + * Apply `batches` one by one, each checked against the state the earlier ones left (`prepareCommit()`, which only + * checks and detaches, so a batch it refuses changed nothing); stops at the first one refused. Memory is durable at + * once; the receipts follow the last batch. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + _context: Context, + ): Promise { + this.assertOpen(); + const seqs: Seq[] = []; + let refused: { readonly error: unknown } | undefined; + try { + for (const writes of batches) { + let prepared: PreparedMemoryCommit; + try { + prepared = this.prepareCommit(writes); + } catch (error) { + refused = { error }; + break; + } + seqs.push(prepared.apply()); + } + } finally { + // Also when a batch's apply throws: the batches applied before it are stored, and their receipts stand. + for (const [index, seq] of seqs.entries()) receive(index, seq); + } + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; } /** Validate and detach one commit without changing observable state. */ diff --git a/packages/durable/src/storage/sqlite/storage.ts b/packages/durable/src/storage/sqlite/storage.ts index 5576363ed..1e55d6c72 100644 --- a/packages/durable/src/storage/sqlite/storage.ts +++ b/packages/durable/src/storage/sqlite/storage.ts @@ -4,6 +4,7 @@ import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "@earendil-work import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, @@ -148,6 +149,31 @@ const writeId = (write: StorageWrite): Id | undefined => { } }; +/** Whether a database call of the checks failed, whatever it threw or rejected with. */ +type CheckCalls = { failed: boolean }; + +/** + * `executor`, setting `calls.failed` when one of its calls throws, synchronously (a synchronous driver) or by + * rejecting, with any value: then a failing check is the database's, not one of Durable's own. + */ +function watchCalls(executor: SqliteExecutor, calls: CheckCalls): SqliteExecutor { + const watched = async (call: () => Promise): Promise => { + try { + return await call(); + } catch (error) { + calls.failed = true; + throw error; + } + }; + return { + exec: (sql) => watched(() => executor.exec(sql)), + run: (sql, ...params) => watched(() => executor.run(sql, ...params)), + runCounted: (sql, ...params) => watched(() => executor.runCounted(sql, ...params)), + get: (sql, ...params) => watched(() => executor.get(sql, ...params)), + all: (sql, ...params) => watched(() => executor.all(sql, ...params)), + }; +} + /** Portable SQLite implementation of the Pico storage contract. */ export class SqliteStorage implements Storage { private readonly db: SqliteDatabase; @@ -184,31 +210,89 @@ export class SqliteStorage implements Storage { } } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, _context: Context): Promise { + /** + * Commit `batches` in one SQL transaction, each with its own seq. Each batch is checked before it writes anything, + * against the state the earlier ones left, and the first one refused ends the batch: the transaction commits the ones + * before it. No savepoints, which Durable Objects do not allow: the checks run before the writes instead. + */ + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + _context: Context, + ): Promise { this.assertOpen(); - const documentActions = this.prepareDocumentActions(writes); - const candidateNextId = this.candidateNextId(writes); - const seq = await this.db.transaction(async (transaction) => { - const metadata = await transaction.get( - `SELECT next_id, next_seq FROM ${this.schema.tables.durableMetadata} WHERE singleton = 1`, - ); - if (metadata === undefined) throw new Error("Durable SQLite metadata is missing"); - const committedSeq = seqFromNumber(metadata.next_seq); - await this.checkGlobalIds(transaction, writes); - await this.checkDocumentActions(transaction, documentActions); - for (const write of writes) await this.applyTableWrite(transaction, write, committedSeq); - await this.applyDocumentActions(transaction, documentActions, committedSeq); - await transaction.run( - `UPDATE ${this.schema.tables.durableMetadata} SET next_id = ?, next_seq = ? WHERE singleton = 1`, - String(Math.max(Number(metadata.next_id), candidateNextId)), - committedSeq + 1, - ); - return committedSeq; + const seqs: Seq[] = []; + let refused: { readonly error: unknown } | undefined; + let candidateNextId = this.nextId; + await this.db.transaction(async (transaction) => { + const metadata = await this.readMetadata(transaction); + let next = metadata.next_seq; + // Whether a database call of a batch's checks failed: then the failure is an unknown outcome, not a refusal. + const calls: CheckCalls = { failed: false }; + const checked = watchCalls(transaction, calls); + for (const writes of batches) { + let documentActions: Map; + // Only Durable's own checks refuse a batch, before it wrote anything: the ones before it stand. A failure of + // the database rejects the call, and rolls back the batches before it with the transaction. + calls.failed = false; + try { + documentActions = this.prepareDocumentActions(writes); + await this.checkBatch(checked, writes, documentActions); + } catch (error) { + if (calls.failed) throw error; + refused = { error }; + break; + } + const seq = seqFromNumber(next++); + await this.applyBatch(transaction, writes, documentActions, seq); + seqs.push(seq); + candidateNextId = Math.max(candidateNextId, this.candidateNextId(writes)); + } + if (seqs.length > 0) { + await this.writeMetadata(transaction, Math.max(Number(metadata.next_id), candidateNextId), next); + } }); this.nextId = Math.max(this.nextId, candidateNextId); - // `COMMIT` returned: the batch is durable. - adopt(seq); - return seq; + // `COMMIT` returned: the stored batches are durable. + for (const [index, seq] of seqs.entries()) receive(index, seq); + return refused === undefined ? { stored: seqs.length } : { stored: seqs.length, refused: refused.error }; + } + + private async readMetadata(executor: SqliteExecutor): Promise { + const metadata = await executor.get( + `SELECT next_id, next_seq FROM ${this.schema.tables.durableMetadata} WHERE singleton = 1`, + ); + if (metadata === undefined) throw new Error("Durable SQLite metadata is missing"); + return metadata; + } + + private async writeMetadata(executor: SqliteExecutor, nextId: number, nextSeq: number): Promise { + await executor.run( + `UPDATE ${this.schema.tables.durableMetadata} SET next_id = ?, next_seq = ? WHERE singleton = 1`, + String(nextId), + nextSeq, + ); + } + + /** Durable's own checks of one batch, copy resolution included, before it writes anything. */ + private async checkBatch( + executor: SqliteExecutor, + writes: readonly StorageWrite[], + documentActions: Map, + ): Promise { + await this.checkGlobalIds(executor, writes); + await this.checkDocumentActions(executor, documentActions); + await this.resolveCopies(executor, documentActions); + } + + private async applyBatch( + executor: SqliteExecutor, + writes: readonly StorageWrite[], + documentActions: ReadonlyMap, + seq: Seq, + ): Promise { + for (const write of writes) await this.applyTableWrite(executor, write, seq); + await this.applyDocumentActions(executor, documentActions, seq); } async mintId>(): Promise { @@ -863,33 +947,39 @@ export class SqliteStorage implements Storage { ); } + /** Resolve each copy to a base of its source's state before this batch, which may not change the source. */ + private async resolveCopies(executor: SqliteExecutor, actions: Map): Promise { + for (const [id, action] of actions) { + if (action.copy === undefined) continue; + try { + const stored = await this.materializeDocument(executor, action.copy.id, action.copy.at); + if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); + const create = action.create!; + if ( + stored.record.scope.kind !== "conversation" || + create.scope.kind !== "conversation" || + stored.record.kind !== create.kind || + stored.record.key !== create.key || + stored.record.history !== create.history || + stored.record.fork !== create.fork + ) { + throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); + } + action.content = { kind: "base", version: stored.version, value: stored.value }; + } catch (error) { + throw new Error(`Document copy ${id} was rejected`, { cause: error }); + } + } + } + private async applyDocumentActions( executor: SqliteExecutor, actions: ReadonlyMap, seq: Seq, ): Promise { for (const [id, action] of actions) { - let content = action.content; - if (action.copy !== undefined) { - try { - const stored = await this.materializeDocument(executor, action.copy.id, action.copy.at); - if (stored === undefined) throw new Error(`Fork source document ${action.copy.id} cannot be read`); - const create = action.create!; - if ( - stored.record.scope.kind !== "conversation" || - create.scope.kind !== "conversation" || - stored.record.kind !== create.kind || - stored.record.key !== create.key || - stored.record.history !== create.history || - stored.record.fork !== create.fork - ) { - throw new Error(`Fork source document ${action.copy.id} does not match the copied record`); - } - content = { kind: "base", version: stored.version, value: stored.value }; - } catch (error) { - throw new Error(`Document copy ${id} was rejected`, { cause: error }); - } - } + // Copies are resolved to their content by `resolveCopies()`. + const content = action.content; let record: DocumentRecord; if (action.create !== undefined) { record = { diff --git a/packages/durable/src/storage/store/commit.ts b/packages/durable/src/storage/store/commit.ts index cc53f5418..7f89e7130 100644 --- a/packages/durable/src/storage/store/commit.ts +++ b/packages/durable/src/storage/store/commit.ts @@ -1,8 +1,9 @@ -// One Durable commit as one composite write of the store's commit step (slice 6b §2.3): staged before the queue (rows -// encoded, contents serialized, explicit ids raising the store's allocator, fork copies resolved from their sources), -// then inside the step what `SqliteStorage` does inside its transaction, with the step's reader: Durable's checks of -// ids and documents (its messages), the table puts, and the documents in two passes, existing ones first (their content, -// then their retirement, which frees the addresses this commit's creations take), then the creations and copies. +// One Durable commit as one commit of the store's series (slice 6b §2.3): staged before the queue (rows encoded, +// contents serialized, explicit ids raising the store's allocator), then inside the series' transaction what +// `SqliteStorage` does inside its own, with the step's reader: Durable's checks of ids and documents and the copies +// resolved from their sources (its messages, a refusal), then its one composite write: the table puts, and the +// documents in two passes, existing ones first (their content, then their retirement, which frees the addresses this +// commit's creations take), then the creations and copies. import type { Context, JsonValue } from "@earendil-works/chord"; import { @@ -14,22 +15,23 @@ import { type OwnerTable, readDocument, retireRecord, + type SeriesCommit, type StagedWrite, type Step, type StoreWrite, writeContent, } from "@earendil-works/chord/next"; +import { StorageRequestError } from "../../errors.ts"; import type { DocumentContent, DocumentCopySource, DocumentCreate, DocumentId, - DocumentRecord, Id, JsonObject, StorageWrite, } from "../../types.ts"; -import { addressOf, durableRecord, engineRecord } from "./documents.ts"; +import { addressOf, durableRecord, engineRecord, readAt } from "./documents.ts"; import { conversationRow, entryRow, submissionRow, taskRow } from "./tables.ts"; /** Durable's record types, as `record_ids` names them. */ @@ -66,14 +68,10 @@ type DocumentAction = { retire: boolean; }; -/** Reads a copy's source at its point: its stored document, `undefined` when it cannot be read there. */ -export type CopySource = ( - source: DocumentCopySource, -) => Promise<{ readonly record: DocumentRecord; readonly version: number; readonly value: JsonObject } | undefined>; - -/** A Durable commit staged for the store's step. */ -export class DurableCommit implements StagedWrite { +/** A Durable commit staged for the store's series: its checks, then itself as its one write. */ +export class DurableCommit implements SeriesCommit, StagedWrite { readonly order = "write"; + readonly writes: readonly StagedWrite[]; readonly #owner: string; readonly #tables: CommitTables; readonly #context: Context; @@ -81,6 +79,10 @@ export class DurableCommit implements StagedWrite { readonly #claims: { readonly id: number; readonly type: RecordType }[] = []; readonly #rows: RowWrite[] = []; readonly #actions = new Map(); + /** What `check` found for `apply`, in the same step: the stored types of the claimed ids, the changed documents. */ + #checked: + | { readonly claimed: Map; readonly existing: Map } + | undefined; /** * Stages `writes` (rule 8: rows and contents encoded here, before the queue); `raise` makes room above each explicit @@ -97,19 +99,35 @@ export class DurableCommit implements StagedWrite { this.#tables = tables; this.#context = context; for (const write of writes) this.#stage(write, raise); + this.writes = [this]; } /** - * Resolves the copies before the step (S11): each source read at its point and checked against the copy's record, - * entered as a creation carrying the source's value as its base. + * Durable's checks against the state the series' earlier commits left, writing nothing: the ids, the documents and + * the copies; a throw refuses the commit. */ - async resolveCopies(read: CopySource): Promise { + async check(step: Step): Promise { + const claimed = await this.#checkIds(step); + const existing = await this.#checkDocuments(step); + await this.#resolveCopies(step); + this.#checked = { claimed, existing }; + } + + /** + * Resolves the copies (S11): each source read at its point and checked against the copy's record, entered as a + * creation carrying the source's value as its base. + */ + async #resolveCopies(step: Step): Promise { for (const [id, action] of this.#actions) { const source = action.copy; if (source === undefined) continue; if (this.#actions.has(source.id)) throw new Error(`Document copy ${id} source is changed in the copy batch`); try { - const stored = await read(source); + const read = await readAt(step, source.id, source.at, this.#owner); + if (read.kind === "current_only") { + throw new StorageRequestError(`Document ${read.id} does not retain historical content`); + } + const stored = read.document; if (stored === undefined) throw new Error(`Fork source document ${source.id} cannot be read`); const create = action.create!; if ( @@ -129,10 +147,10 @@ export class DurableCommit implements StagedWrite { } } + /** The commit's writes, after its `check` in the same step. */ async apply(step: Step): Promise { const recorded: StoreWrite[] = []; - const claimed = await this.#checkIds(step); - const existing = await this.#checkDocuments(step); + const { claimed, existing } = this.#checked!; for (const write of this.#rows) { if (claimed.get(write.id) === undefined) recorded.push(await this.#claim(step, write.id, write.type)); claimed.set(write.id, write.type); diff --git a/packages/durable/src/storage/store/storage.ts b/packages/durable/src/storage/store/storage.ts index 554979b89..bfbfb894c 100644 --- a/packages/durable/src/storage/store/storage.ts +++ b/packages/durable/src/storage/store/storage.ts @@ -8,6 +8,7 @@ import { type Claimant, findDocumentEntry, listDocumentEntries, + type SeriesCommit, type Store, StoreOwnerTables, } from "@earendil-works/chord/next"; @@ -15,12 +16,12 @@ import type { SqliteExecutor } from "@earendil-works/chord/sqlite"; import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { + CommitResult, ConversationId, ConversationQuery, ConversationRecord, Cursor, DocumentAddress, - DocumentCopySource, DocumentId, DocumentPoint, DocumentQuery, @@ -84,8 +85,8 @@ function cursorId(cursor: Cursor | undefined): number | undefined { /** * Durable's `Storage` on Chord's store (slice 6b D1, D2, §2): its records as declared tables under an owner name, its - * documents in the store's documents engine, its ids from the store's one allocator, its commits as one composite - * write of the store's commit step, adopted in the host's batch. Several may share a store under distinct owner names + * documents in the store's documents engine, its ids from the store's one allocator, each commit call as one series + * of the store's commit queue (one backend transaction, one seq per batch), published in the host's batch. Several may share a store under distinct owner names * (claims); the app owns the store, and `close` leaves it open. */ export class StoreStorage implements Storage { @@ -144,17 +145,43 @@ export class StoreStorage implements Storage { } } - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise { - return this.#call(async () => { + /** + * Commit `batches` as one series of the store (`ChordStore.commitSeries`): one backend transaction, one seq per + * batch, each batch's checks (at staging, then against the state the earlier ones left) before its writes. + */ + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise { + return this.#call(() => { const store = this.#store; - const commit = new DurableCommit(writes, this.#owner, this.#tables, (id) => store.ids.raise(id), context); - await commit.resolveCopies((source) => this.#copySource(source, context)); - // An empty commit stages nothing: it takes no seq, adopted at `lastSeq`. - const staged = writes.length === 0 ? [] : [commit]; - return seqFromNumber(await store.commitInBatch(staged, (seq) => adopt(seqFromNumber(seq)), context)); + const raise = (id: number): void => store.ids.raise(id); + const commits: SeriesCommit[] = []; + for (const writes of batches) { + try { + commits.push(new DurableCommit(writes, this.#owner, this.#tables, raise, context)); + } catch (error) { + // Staging runs Durable's checks of the batch itself: a refusal in its turn, the batches before it stored. + commits.push({ check: () => Promise.reject(error), writes: [] }); + break; + } + } + return store.commitSeries(commits, (index, seq) => receive(index, seqFromNumber(seq)), context); }); } + /** + * Run a commit's publication in the attached host's batch (`batch.within`: it joins a running batch, opens one, or + * runs next while one sends): Chord sources over Durable's views are marked in one batch per commit. Plainly without + * a host. + */ + publishing(publish: () => void, context: Context): void { + const batch = this.#store.batch; + if (batch === undefined) publish(); + else batch.within(publish, context); + } + mintId>(): Promise { return this.#call(async () => idFromNumber(this.#store.mintId())); } @@ -525,15 +552,6 @@ export class StoreStorage implements Storage { return page(values, limit, start.order); }); } - - /** A copy's source read at its point, before the step (S11); a request error is thrown here, outside the read. */ - async #copySource(source: DocumentCopySource, context: Context) { - const read = await this.#store.readTransaction( - (executor) => readAt(executor, source.id, source.at, this.#owner), - context, - ); - return documentOf(read); - } } /** A read that found nothing. */ diff --git a/packages/durable/src/testing/commit-writes.ts b/packages/durable/src/testing/commit-writes.ts new file mode 100644 index 000000000..7d655294d --- /dev/null +++ b/packages/durable/src/testing/commit-writes.ts @@ -0,0 +1,18 @@ +// One batch through `Storage.commit()`, for tests and suites that seed or write a storage directly. + +import type { Context } from "@earendil-works/chord"; +import type { Seq, Storage, StorageWrite } from "../types.ts"; + +/** Commits `writes` as one batch and resolves with its seq; rejects with the refusal if the storage refused it. */ +export async function commitWrites(storage: Storage, writes: readonly StorageWrite[], context: Context): Promise { + let stored: Seq | undefined; + const result = await storage.commit( + [writes], + (_index, seq) => { + stored = seq; + }, + context, + ); + if (result.stored === 0) throw result.refused; + return stored!; +} diff --git a/packages/durable/src/testing/index.ts b/packages/durable/src/testing/index.ts index fdaa421f5..394fd24eb 100644 --- a/packages/durable/src/testing/index.ts +++ b/packages/durable/src/testing/index.ts @@ -1,4 +1,5 @@ export { createExpectAssertions, type ExpectLike } from "./assertions.ts"; +export { commitWrites } from "./commit-writes.ts"; export { createEnvConformance } from "./env-conformance.ts"; export { registerEnvConformance, registerStorageConformance, type StorageConformanceRunner } from "./runner.ts"; export { diff --git a/packages/durable/src/testing/storage-benchmark.ts b/packages/durable/src/testing/storage-benchmark.ts index d8e0e3ad4..9acef51c9 100644 --- a/packages/durable/src/testing/storage-benchmark.ts +++ b/packages/durable/src/testing/storage-benchmark.ts @@ -13,9 +13,7 @@ import type { TaskRecord, } from "../types.ts"; import { ROOT_CONVERSATION_ID } from "../types.ts"; - -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; +import { commitWrites } from "./commit-writes.ts"; export type StorageBenchmarkScale = { readonly name: string; @@ -92,7 +90,7 @@ export async function seedStorageBenchmark( storage: Storage, scale: StorageBenchmarkScale = TIMING_SCALE, ): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); let firstEntryId: EntryId | undefined; for (let start = 0; start < scale.entryCount; start += BATCH_SIZE) { @@ -111,7 +109,7 @@ export async function seedStorageBenchmark( }, }); } - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); } for (let start = 0; start < scale.taskCount; start += BATCH_SIZE) { @@ -119,7 +117,7 @@ export async function seedStorageBenchmark( for (let index = start; index < Math.min(start + BATCH_SIZE, scale.taskCount); index++) { writes.push({ type: "task", value: task(await storage.mintId>(), index) }); } - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); } let exactDocumentId: DocumentId | undefined; @@ -134,7 +132,7 @@ export async function seedStorageBenchmark( content: { kind: "base", version: 1, value: { index, text: "x".repeat(128) } }, }); } - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); } const replayEntries = await Promise.all( @@ -143,7 +141,8 @@ export async function seedStorageBenchmark( return { tail, id }; }), ); - await storage.commit( + await commitWrites( + storage, replayEntries.map(({ tail, id }) => ({ type: "document.create" as const, record: { @@ -156,11 +155,11 @@ export async function seedStorageBenchmark( }, content: { kind: "base" as const, version: 1, value: { count: 0, text: "x".repeat(64) } }, })), - adopt, BACKGROUND_CONTEXT, ); for (let count = 1; count <= REPLAY_TAILS.at(-1)!; count++) { - await storage.commit( + await commitWrites( + storage, replayEntries .filter(({ tail }) => count <= tail) .map(({ id }) => ({ @@ -168,7 +167,6 @@ export async function seedStorageBenchmark( id, content: { kind: "delta" as const, version: 1, ops: [["s", ["count"], count] as const] }, })), - adopt, BACKGROUND_CONTEXT, ); } @@ -181,7 +179,8 @@ export async function seedStorageBenchmark( history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -189,12 +188,12 @@ export async function seedStorageBenchmark( content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, BACKGROUND_CONTEXT, ); let ancientAt: Seq | undefined; for (let count = 1; count <= HISTORY_SEGMENT_LENGTH; count++) { - ancientAt = await storage.commit( + ancientAt = await commitWrites( + storage, [ { type: "document.change", @@ -202,11 +201,11 @@ export async function seedStorageBenchmark( content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, BACKGROUND_CONTEXT, ); } - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -214,13 +213,13 @@ export async function seedStorageBenchmark( content: { kind: "base", version: 1, value: { count: HISTORY_SEGMENT_LENGTH } }, }, ], - adopt, BACKGROUND_CONTEXT, ); if (ancientAt === undefined) throw new Error("Benchmark history seed produced no commits"); let recentAt = ancientAt; for (let count = HISTORY_SEGMENT_LENGTH + 1; count <= HISTORY_SEGMENT_LENGTH * 2; count++) { - recentAt = await storage.commit( + recentAt = await commitWrites( + storage, [ { type: "document.change", @@ -228,7 +227,6 @@ export async function seedStorageBenchmark( content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, BACKGROUND_CONTEXT, ); } @@ -239,18 +237,19 @@ export async function seedStorageBenchmark( let deepestConversationId = ROOT_CONVERSATION_ID; for (let depth = 0; depth < FORK_DEPTH; depth++) { const conversationId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: conversationId, parent: { conversationId: parentConversationId, at: parentAt } }, }, ], - adopt, BACKGROUND_CONTEXT, ); const ids = await Promise.all(Array.from({ length: ENTRIES_PER_FORK }, () => storage.mintId())); - await storage.commit( + await commitWrites( + storage, ids.map((id, index) => ({ type: "entry" as const, value: { @@ -260,7 +259,6 @@ export async function seedStorageBenchmark( data: { depth, index }, }, })), - adopt, BACKGROUND_CONTEXT, ); parentConversationId = conversationId; @@ -400,8 +398,9 @@ export type StorageWriteBenchmark = { /** Seed the common state expected by every write benchmark sample. */ export async function seedStorageWriteBenchmark(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT); - await storage.commit( + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); + await commitWrites( + storage, await Promise.all( Array.from({ length: 100 }, async (_, index) => ({ type: "entry" as const, @@ -413,7 +412,6 @@ export async function seedStorageWriteBenchmark(storage: Storage): Promise }, })), ), - adopt, BACKGROUND_CONTEXT, ); } @@ -424,7 +422,8 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ expected: 1, async run(storage) { const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -436,7 +435,6 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ }, }, ], - adopt, BACKGROUND_CONTEXT, ); return 1; @@ -460,7 +458,7 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ }), ), ); - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); return writes.length; }, }, @@ -495,7 +493,7 @@ export const STORAGE_WRITE_BENCHMARKS: readonly StorageWriteBenchmark[] = [ content: { kind: "base", version: 1, value: { entryId, taskId } }, }, ]; - await storage.commit(writes, adopt, BACKGROUND_CONTEXT); + await commitWrites(storage, writes, BACKGROUND_CONTEXT); return writes.length; }, }, diff --git a/packages/durable/src/testing/storage-conformance.ts b/packages/durable/src/testing/storage-conformance.ts index 91b9cf057..70f295454 100644 --- a/packages/durable/src/testing/storage-conformance.ts +++ b/packages/durable/src/testing/storage-conformance.ts @@ -22,16 +22,14 @@ import { type TaskId, type TaskRecord, } from "../types.ts"; +import { commitWrites } from "./commit-writes.ts"; import type { StorageConformanceAssertions, StorageConformanceCase, StorageConformanceOptions } from "./types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - const context = BACKGROUND_CONTEXT; type StoredTask = TaskRecord; async function createRoot(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); return ROOT_CONVERSATION_ID; } @@ -92,30 +90,27 @@ function assertionFacade(assertions: StorageConformanceAssertions) { } /** - * `storage` with every commit checked against the adoption contract: `adopt` runs exactly once, with the committed - * sequence, before the commit resolves (so before the next commit starts), and never for a commit that rejects. The - * cases fail commits only before they are durable. + * `storage` with every commit checked against the receipt contract: one `receive` per stored batch, in order, with + * increasing seqs, all before the call settles. */ -function checkAdoption(storage: Storage, assertions: StorageConformanceAssertions): Storage { - const commit: Storage["commit"] = async (writes, adopt, commitContext) => { - const adopted: Seq[] = []; +function checkReceipts(storage: Storage, assertions: StorageConformanceAssertions): Storage { + const commit: Storage["commit"] = async (batches, receive, commitContext) => { + const seqs: Seq[] = []; let settled = false; - const recording = (seq: Seq): void => { - assertions.ok(!settled, "Storage.commit() adopted after it settled"); - adopted.push(seq); - adopt(seq); + const recording = (index: number, seq: Seq): void => { + assertions.ok(!settled, "Storage.commit() received a batch after it settled"); + assertions.strictEqual(index, seqs.length); + if (seqs.length > 0) assertions.greaterThan(seq, seqs.at(-1)!); + seqs.push(seq); + receive(index, seq); }; - let seq: Seq; try { - seq = await storage.commit(writes, recording, commitContext); - } catch (error) { + const result = await storage.commit(batches, recording, commitContext); + assertions.strictEqual(seqs.length, result.stored); + return result; + } finally { settled = true; - assertions.deepEqual(adopted, []); - throw error; } - settled = true; - assertions.deepEqual(adopted, [seq]); - return seq; }; return new Proxy(storage, { get(target, property) { @@ -127,7 +122,7 @@ function checkAdoption(storage: Storage, assertions: StorageConformanceAssertion } function createCase(options: StorageConformanceOptions, name: string, test: ConformanceTest): StorageConformanceCase { - return { name, run: () => options.withStorage((storage) => test(checkAdoption(storage, options.assertions))) }; + return { name, run: () => options.withStorage((storage) => test(checkReceipts(storage, options.assertions))) }; } /** Creates runner-independent cases. `withStorage` must call and await its callback exactly once per case. */ @@ -139,50 +134,51 @@ export function createStorageConformance(options: StorageConformanceOptions): re await expect(createRoot(storage)).resolves.toBe(ROOT_CONVERSATION_ID); expect(await storage.conversation(ROOT_CONVERSATION_ID, context)).toEqual({ id: ROOT_CONVERSATION_ID }); await expect( - storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context), + commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context), ).rejects.toThrow(`ID ${ROOT_CONVERSATION_ID} already belongs to conversation`); }), - createCase(options, "adopts a durable commit once before resolving, and never a failed one", async (storage) => { - const rootId = await createRoot(storage); - const events: string[] = []; - const entryId = await storage.mintId(); - let read: ReturnType | undefined; - const seq = await storage - .commit( - [{ type: "entry", value: entry(entryId, rootId) }], - (adopted) => { - events.push(`adopt ${adopted}`); - // The batch is durable when it is adopted: a read started now observes it. + createCase( + options, + "receives a durable commit once before resolving, and never a refused one", + async (storage) => { + const rootId = await createRoot(storage); + const events: string[] = []; + const entryId = await storage.mintId(); + let read: ReturnType | undefined; + let seq: Seq | undefined; + const result = await storage.commit( + [[{ type: "entry", value: entry(entryId, rootId) }]], + (_index, received) => { + seq = received; + events.push(`receive ${received}`); + // The batch is durable when it is received: a read started now observes it. read = storage.entry(entryId, context); }, context, - ) - .then((committed) => { - events.push(`resolve ${committed}`); - return committed; - }); - expect(events).toEqual([`adopt ${seq}`, `resolve ${seq}`]); - expect((await read)?.commitSeq).toBe(seq); + ); + events.push(`resolve ${result.stored}`); + expect(events).toEqual([`receive ${seq}`, "resolve 1"]); + expect((await read)?.commitSeq).toBe(seq); - events.length = 0; - await expect( - storage.commit( - [{ type: "conversation", value: { id: rootId } }], - (adopted) => events.push(`adopt ${adopted}`), + events.length = 0; + const refused = await storage.commit( + [[{ type: "conversation", value: { id: rootId } }]], + (_index, received) => events.push(`receive ${received}`), context, - ), - ).rejects.toThrow(`ID ${rootId} already belongs to conversation`); - expect(events).toEqual([]); + ); + expect(refused.stored).toBe(0); + expect(String(refused.refused)).toBe(`Error: ID ${rootId} already belongs to conversation`); + expect(events).toEqual([]); - const next = await storage.commit( - [{ type: "entry", value: entry(await storage.mintId(), rootId) }], - (adopted) => events.push(`adopt ${adopted}`), - context, - ); - expect(events).toEqual([`adopt ${next}`]); - expect(next).toBeGreaterThan(seq); - }), + const next = await commitWrites( + storage, + [{ type: "entry", value: entry(await storage.mintId(), rootId) }], + context, + ); + expect(next).toBeGreaterThan(seq!); + }, + ), createCase( options, @@ -201,13 +197,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re status: "placed", entry: entryId, }; - const initialSeq = await storage.commit( + const initialSeq = await commitWrites( + storage, [ { type: "entry", value: entry(entryId, rootId, "user", { data: { text: "hello" } }) }, { type: "task", value: task }, { type: "submission", value: input }, ], - adopt, context, ); @@ -225,14 +221,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re }; const doneInput: SubmissionRecord = { ...input, status: "done", answer: transientEntryId }; await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", value: runningTask }, { type: "submission", value: doneInput }, { type: "entry", value: entry(transientEntryId, rootId, "assistant") }, { type: "conversation", value: { id: rootId } }, ], - adopt, context, ), ).rejects.toThrow(`ID ${rootId} already belongs to conversation`); @@ -240,9 +236,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect(await storage.task(taskId, context)).toEqual(task); expect(await storage.submission(submissionId, context)).toEqual(input); expect(await storage.entry(transientEntryId, context)).toBeUndefined(); - const afterRollbackSeq = await storage.commit( + const afterRollbackSeq = await commitWrites( + storage, [{ type: "entry", value: entry(await storage.mintId(), rootId, "after-rollback") }], - adopt, context, ); expect(afterRollbackSeq).toBeGreaterThan(initialSeq); @@ -270,13 +266,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re reason: "failed", detail, }; - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: storedEntry }, { type: "task", value: storedTask }, { type: "submission", value: storedInput }, ], - adopt, context, ); @@ -313,7 +309,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re const data = JSON.parse( '{"__proto__":{"polluted":false},"constructor":{"label":"stored"},"toString":"value"}', ) as Record; - await storage.commit([{ type: "entry", value: entry(entryId, rootId, "note", { data }) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(entryId, rootId, "note", { data }) }], context); (Reflect.get(data, "__proto__") as Record).polluted = true; (Reflect.get(data, "constructor") as Record).label = "mutated"; @@ -334,7 +330,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re createCase(options, "indexes entries committed out of ID order", async (storage) => { const rootId = await createRoot(storage); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: entry(idFromNumber(30), rootId) }, { type: "entry", value: entry(idFromNumber(10), rootId) }, @@ -345,7 +342,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }), }, ], - adopt, context, ); @@ -360,20 +356,20 @@ export function createStorageConformance(options: StorageConformanceOptions): re const oldestId = await storage.mintId(); const middleId = await storage.mintId(); const newestId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: entry(oldestId, rootId) }, { type: "entry", value: entry(middleId, rootId) }, { type: "entry", value: entry(newestId, rootId) }, ], - adopt, context, ); const first = await storage.scanEntries({ conversationId: rootId }, 2, undefined, context); expect(first.items.map(({ id }) => id)).toEqual([newestId, middleId]); const appendedId = await storage.mintId(); - await storage.commit([{ type: "entry", value: entry(appendedId, rootId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(appendedId, rootId) }], context); const second = await storage.scanEntries({ conversationId: rootId }, 2, first.next, context); expect(second.items.map(({ id }) => id)).toEqual([oldestId]); expect(second.next).toBeUndefined(); @@ -383,12 +379,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re const rootId = await createRoot(storage); const secondId = await storage.mintId(); const thirdId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: thirdId } }, { type: "conversation", value: { id: secondId } }, ], - adopt, context, ); @@ -409,7 +405,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const firstId = await storage.mintId(); const secondId = await storage.mintId(); const thirdId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: otherOwnerId } }, { @@ -425,7 +422,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re value: { id: thirdId, owner: { conversationId: otherOwnerId, taskId: firstTaskId } }, }, ], - adopt, context, ); @@ -457,7 +453,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const rootFirst = await storage.mintId(); const rootForkPoint = await storage.mintId(); const rootExcludedSameCommit = await storage.mintId(); - const rootEntriesSeq = await storage.commit( + const rootEntriesSeq = await commitWrites( + storage, [ { type: "entry", value: entry(rootFirst, rootId) }, { @@ -466,46 +463,46 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "entry", value: entry(rootExcludedSameCommit, rootId) }, ], - adopt, context, ); const childId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: childId, parent: { conversationId: rootId, at: rootForkPoint } }, }, ], - adopt, context, ); const childForkPoint = await storage.mintId(); const childExcluded = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: entry(childForkPoint, childId, "note") }, { type: "entry", value: entry(childExcluded, childId) }, ], - adopt, context, ); const rootExcludedLater = await storage.mintId(); - await storage.commit([{ type: "entry", value: entry(rootExcludedLater, rootId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(rootExcludedLater, rootId) }], context); const grandchildId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: grandchildId, parent: { conversationId: childId, at: childForkPoint } }, }, ], - adopt, context, ); const grandchildHead = await storage.mintId(); const grandchildTail = await storage.mintId(); - const grandchildEntriesSeq = await storage.commit( + const grandchildEntriesSeq = await commitWrites( + storage, [ { type: "entry", @@ -513,11 +510,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "entry", value: entry(grandchildTail, grandchildId) }, ], - adopt, context, ); const childExcludedLater = await storage.mintId(); - await storage.commit([{ type: "entry", value: entry(childExcludedLater, childId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(childExcludedLater, childId) }], context); const first = await storage.scanEntries({ conversationId: grandchildId }, 2, undefined, context); expect(first.items.map(({ id }) => id)).toEqual([grandchildTail, grandchildHead]); @@ -628,7 +624,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const conversationId = await storage.mintId(); const taskId = await storage.mintId>(); const submissionId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: conversationId } }, { type: "task", value: pendingTask(taskId, rootId) }, @@ -637,7 +634,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re value: { id: submissionId, conversationId: rootId, type: "input", status: "queued" }, }, ], - adopt, context, ); conversationIds.push(conversationId); @@ -680,13 +676,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re const first = { ...pendingTask(firstId, rootId), memos: { winner: "first" } } satisfies StoredTask; const second = { ...pendingTask(secondId, rootId), background: true } satisfies StoredTask; const third = { ...pendingTask(thirdId, rootId), abortRequested: true } satisfies StoredTask; - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: first }, { type: "task", value: second }, { type: "task", value: third }, ], - adopt, context, ); @@ -695,7 +691,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re state: { status: "running", checkpoint: { phase: "effect", attempt: 1 } }, abortRequested: true, }; - await storage.commit([{ type: "task", value: running }], adopt, context); + await commitWrites(storage, [{ type: "task", value: running }], context); expect(await storage.task(firstId, context)).toEqual(running); const terminal: StoredTask = { id: firstId, @@ -707,7 +703,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re background: false, abortRequested: true, }; - await storage.commit([{ type: "task", value: terminal }], adopt, context); + await commitWrites(storage, [{ type: "task", value: terminal }], context); expect(await storage.task(firstId, context)).toEqual(terminal); const pendingPage = await storage.scanTasks({ status: "pending" }, 1, undefined, context); @@ -743,7 +739,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re state: { status: "completing", outcome: { status: "failed", error: { message: "held" } } }, }; const writes = [owner, waiting, completing].map((value) => ({ type: "task", value }) as const); - await storage.commit(writes, adopt, context); + await commitWrites(storage, writes, context); expect(await storage.task(waitingId, context)).toEqual(waiting); expect(await storage.task(completingId, context)).toEqual(completing); const scan = async (status: StoredTask["state"]["status"]) => @@ -755,7 +751,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re ...completing, state: { status: "terminal", outcome: completing.state.outcome! }, }; - await storage.commit([{ type: "task", value: terminal }], adopt, context); + await commitWrites(storage, [{ type: "task", value: terminal }], context); expect(await scan("completing")).toEqual([]); expect(await scan("terminal")).toEqual([terminal]); }), @@ -768,13 +764,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re state: { status: "sleeping", checkpoint: { phase: "next" }, until: 61_000 }, memos: { kept: true }, }; - await storage.commit([{ type: "task", value: sleeping }], adopt, context); + await commitWrites(storage, [{ type: "task", value: sleeping }], context); expect(await storage.task(sleepingId, context)).toEqual(sleeping); const scan = async (status: StoredTask["state"]["status"]) => (await storage.scanTasks({ status }, 10, undefined, context)).items; expect(await scan("sleeping")).toEqual([sleeping]); const running: StoredTask = { ...sleeping, state: { status: "running", checkpoint: { phase: "next" } } }; - await storage.commit([{ type: "task", value: running }], adopt, context); + await commitWrites(storage, [{ type: "task", value: running }], context); expect(await scan("sleeping")).toEqual([]); expect(await scan("running")).toEqual([running]); }), @@ -785,7 +781,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re async (storage) => { const rootId = await createRoot(storage); const secondConversationId = await storage.mintId(); - await storage.commit([{ type: "conversation", value: { id: secondConversationId } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: secondConversationId } }], context); const firstId = await storage.mintId(); const secondId = await storage.mintId(); const otherConversationId = await storage.mintId(); @@ -810,13 +806,13 @@ export function createStorageConformance(options: StorageConformanceOptions): re type: "input", status: "queued", }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: first }, { type: "submission", value: second }, { type: "submission", value: otherConversation }, ], - adopt, context, ); expect(await storage.submissionByRequest(rootId, "same", context)).toEqual(first); @@ -827,7 +823,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re status: "placed", entry: await storage.mintId(), }; - await storage.commit([{ type: "submission", value: placedSecond }], adopt, context); + await commitWrites(storage, [{ type: "submission", value: placedSecond }], context); expect(await storage.submission(secondId, context)).toEqual(placedSecond); expect(await storage.submissionByRequest(rootId, "other", context)).toEqual(placedSecond); @@ -874,12 +870,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re type: "write", status: "queued", }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: queuedDone }, { type: "submission", value: queuedFailed }, ], - adopt, context, ); @@ -894,12 +890,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re reason: "closed", detail: { retryable: false }, }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: done }, { type: "submission", value: unanswered }, ], - adopt, context, ); expect(await storage.submission(doneId, context)).toEqual(done); @@ -926,12 +922,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re type: "continue", status: "placed", }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: placedDone }, { type: "submission", value: placedFailed }, ], - adopt, context, ); expect(await storage.submission(doneId, context)).toEqual(placedDone); @@ -942,12 +938,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re const done: SubmissionRecord = { ...placedDone, status: "done", answer: await storage.mintId() }; const unanswered: SubmissionRecord = { ...placedFailed, status: "unanswered", reason: "nothing_to_answer" }; - await storage.commit( + await commitWrites( + storage, [ { type: "submission", value: done }, { type: "submission", value: unanswered }, ], - adopt, context, ); expect(await storage.submission(doneId, context)).toEqual(done); @@ -967,7 +963,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re fork: "asOf", } satisfies DocumentCreate; const initial: JsonObject = { items: ["a"], nested: { count: 1 } }; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "document.create", @@ -975,7 +972,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: initial }, }, ], - adopt, context, ); const appended = ["b"]; @@ -983,9 +979,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re ["p", ["items"], 1, 0, appended], ["s", ["nested", "count"], 2], ]; - const changedAt = await storage.commit( + const changedAt = await commitWrites( + storage, [{ type: "document.change", id: firstId, content: { kind: "delta", version: 1, ops } }], - adopt, context, ); @@ -1005,7 +1001,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re nested: { count: 2 }, }); - const checkpointAt = await storage.commit( + const checkpointAt = await commitWrites( + storage, [ { type: "document.change", @@ -1013,10 +1010,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 2, value: { items: ["checkpoint"], nested: { count: 3 } } }, }, ], - adopt, context, ); - const replacedAt = await storage.commit( + const replacedAt = await commitWrites( + storage, [ { type: "document.change", @@ -1028,7 +1025,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); expect(await storage.document(firstId, changedAt, context)).toMatchObject({ @@ -1047,7 +1043,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect((await storage.document(firstId, "current", context))?.deltasSinceBase).toBe(1); const secondId = await storage.mintId(); - const retiredAt = await storage.commit( + const retiredAt = await commitWrites( + storage, [ { type: "document.create", @@ -1061,7 +1058,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 2, ops: [["s", ["retiring"], true]] }, }, ], - adopt, context, ); const address = { kind: firstRecord.kind, scope: firstRecord.scope }; @@ -1098,9 +1094,9 @@ export function createStorageConformance(options: StorageConformanceOptions): re revision: 0, rows: Array.from({ length: 512 }, (_, value) => ({ value, stable: `row-${value}` })), }; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: initial } }], - adopt, context, ); const beforeReplacement = structuredClone(initial); @@ -1109,7 +1105,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const index = (revision * 17) % beforeReplacement.rows.length; beforeReplacement.rows[index]!.value = -revision; beforeReplacement.revision = revision; - beforeReplacementAt = await storage.commit( + beforeReplacementAt = await commitWrites( + storage, [ { type: "document.change", @@ -1124,7 +1121,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); } @@ -1134,7 +1130,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re rows: Array.from({ length: 512 }, (_, value) => ({ value: 10_000 + value, stable: `new-${value}` })), }; const replacementSnapshot = structuredClone(replacement); - const replacementAt = await storage.commit( + const replacementAt = await commitWrites( + storage, [ { type: "document.change", @@ -1142,7 +1139,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 1, ops: [["r", replacement]] }, }, ], - adopt, context, ); replacement.rows[0]!.value = -999; @@ -1152,7 +1148,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const index = (revision * 19) % current.rows.length; current.rows[index]!.value = -revision; current.revision = revision; - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1167,7 +1164,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); } @@ -1188,12 +1184,12 @@ export function createStorageConformance(options: StorageConformanceOptions): re const rootId = await createRoot(storage); const childId = await storage.mintId(); const secondChildId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: childId } }, { type: "conversation", value: { id: secondChildId } }, ], - adopt, context, ); const sourceId = await storage.mintId(); @@ -1204,7 +1200,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "document.create", @@ -1212,10 +1209,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 2, value: { count: 1, rows: [{ value: "base" }] } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1230,7 +1227,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, }, ], - adopt, context, ); const historicalCopyId = await storage.mintId(); @@ -1243,7 +1239,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re history: "rewindable", fork: "asOf", }); - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1262,7 +1259,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: retiredCopyId }, ], - adopt, context, ); expect(await storage.document(historicalCopyId, "current", context)).toMatchObject({ @@ -1275,7 +1271,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re }); expect(await storage.document(retiredCopyId, "current", context)).toBeUndefined(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1284,7 +1281,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: sourceId }, ], - adopt, context, ); expect((await storage.document(currentCopyId, "current", context))?.value).toEqual({ @@ -1301,7 +1297,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re history: "latest", fork: "current", } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1309,10 +1306,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 4, value: { retained: "copy" } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1324,10 +1321,10 @@ export function createStorageConformance(options: StorageConformanceOptions): re source: { id: latestSourceId, at: "current" }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1336,7 +1333,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: latestSourceId }, ], - adopt, context, ); expect(await storage.document(latestCopyId, "current", context)).toMatchObject({ @@ -1347,7 +1343,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const conflictId = await storage.mintId(); let conflictError: unknown; try { - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1356,7 +1353,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: currentCopyId }, ], - adopt, context, ); } catch (error) { @@ -1373,7 +1369,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const mismatchId = await storage.mintId(); let mismatchError: unknown; try { - await storage.commit( + await commitWrites( + storage, [ { type: "document.copy", @@ -1381,7 +1378,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re source: { id: currentCopyId, at: "current" }, }, ], - adopt, context, ); } catch (error) { @@ -1403,19 +1399,19 @@ export function createStorageConformance(options: StorageConformanceOptions): re kind: "session.settings", scope: { kind: "session" }, } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 1 } } }], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] } }], - adopt, context, ); - const migratedAt = await storage.commit( + const migratedAt = await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 2, value: { count: 3 } } }], - adopt, context, ); expect(await storage.document(id, "current", context)).toMatchObject({ version: 2, value: { count: 3 } }); @@ -1424,14 +1420,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re ); await expect( - storage.commit( + commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], 4]] } }], - adopt, context, ), ).rejects.toThrow("version transition requires a base"); expect((await storage.document(id, "current", context))?.value).toEqual({ count: 3 }); - await storage.commit([{ type: "document.retire", id }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id }], context); expect(await storage.document(id, "current", context)).toBeUndefined(); }, ), @@ -1445,7 +1441,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const taskSingletonId = await storage.mintId(); const taskFamilyId = await storage.mintId(); const taskOtherKindId = await storage.mintId(); - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId, rootId) }, { @@ -1509,7 +1506,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: { owner: "other" } }, }, ], - adopt, context, ); @@ -1588,13 +1584,14 @@ export function createStorageConformance(options: StorageConformanceOptions): re kind: "singleton", scope: { kind: "session" }, } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { value: 1 } } }], - adopt, context, ); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.create", @@ -1603,7 +1600,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.change", id: firstId, content: { kind: "delta", version: 1, ops: [] } }, ], - adopt, context, ), ).rejects.toThrow("already has a current incarnation"); @@ -1611,7 +1607,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect(await storage.document(secondId, "current", context)).toBeUndefined(); const emptyId = await storage.mintId(); - const emptyAt = await storage.commit( + const emptyAt = await commitWrites( + storage, [ { type: "document.create", @@ -1627,7 +1624,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re }, { type: "document.retire", id: emptyId }, ], - adopt, context, ); expect(await storage.document(emptyId, "current", context)).toBeUndefined(); @@ -1664,7 +1660,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 1, ops: [["s", ["x"], 2]] }, }; const writes = order === "create first" ? [create, change] : [change, create]; - await expect(storage.commit(writes, adopt, context)).rejects.toThrow("more than one content command"); + await expect(commitWrites(storage, writes, context)).rejects.toThrow("more than one content command"); expect(await storage.document(id, "current", context)).toBeUndefined(); } }, @@ -1691,20 +1687,21 @@ export function createStorageConformance(options: StorageConformanceOptions): re kind: "atomic", scope: { kind: "session" }, } satisfies DocumentCreate; - const baselineSeq = await storage.commit( + const baselineSeq = await commitWrites( + storage, [ { type: "task", value: task }, { type: "submission", value: submission }, { type: "document.create", record, content: { kind: "base", version: 1, value: { count: 1 } } }, ], - adopt, context, ); const entryId = await storage.mintId(); const conflictingDocumentId = await storage.mintId(); await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", @@ -1721,7 +1718,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).rejects.toThrow("already has a current incarnation"); @@ -1734,7 +1730,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re expect( (await storage.findDocument({ kind: record.kind, scope: record.scope }, "current", context))?.id, ).toBe(documentId); - const afterRollbackSeq = await storage.commit( + const afterRollbackSeq = await commitWrites( + storage, [ { type: "document.change", @@ -1742,7 +1739,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] }, }, ], - adopt, context, ); expect(afterRollbackSeq).toBeGreaterThan(baselineSeq); @@ -1761,7 +1757,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re const secondKindDocumentId = await storage.mintId(); const firstKeyDocumentId = await storage.mintId(); const secondKeyDocumentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: { ...pendingTask(firstTaskId, rootId), kind: first } }, { type: "task", value: { ...pendingTask(secondTaskId, rootId), kind: second } }, @@ -1806,7 +1803,6 @@ export function createStorageConformance(options: StorageConformanceOptions): re content: { kind: "base", version: 1, value: { identity: "second key" } }, }, ], - adopt, context, ); @@ -1855,30 +1851,248 @@ export function createStorageConformance(options: StorageConformanceOptions): re createCase(options, "keeps one global record ID namespace and rejects exhausted ID minting", async (storage) => { const rootId = await createRoot(storage); const explicitEntryId = idFromNumber(100); - await storage.commit([{ type: "entry", value: entry(explicitEntryId, rootId) }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry(explicitEntryId, rootId) }], context); expect(await storage.mintId()).toBe(101); await expect( - storage.commit( + commitWrites( + storage, [{ type: "task", value: pendingTask(idFromNumber>(explicitEntryId), rootId) }], - adopt, context, ), ).rejects.toThrow(`ID ${explicitEntryId} already belongs to entry`); - await storage.commit( + await commitWrites( + storage, [{ type: "entry", value: entry(idFromNumber(Number.MAX_SAFE_INTEGER), rootId, "last-id") }], - adopt, context, ); await expect(storage.mintId()).rejects.toThrow("ID space is exhausted"); await expect(storage.mintId()).rejects.toThrow("ID space is exhausted"); }), + createCase( + options, + "commits batches in order, each with its seq, checked against the earlier ones", + async (storage) => { + const rootId = await createRoot(storage); + const firstId = await storage.mintId(); + const lastId = await storage.mintId(); + const documentId = await storage.mintId(); + const receipts: [number, Seq][] = []; + let read: ReturnType | undefined; + const result = await storage.commit( + [ + [ + { type: "entry", value: entry(firstId, rootId) }, + { + type: "document.create", + record: { + id: documentId, + kind: "batch.doc", + scope: { kind: "conversation", conversationId: rootId }, + history: "latest", + fork: "initial", + }, + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + // Changes the document the batch before created. + [ + { + type: "document.change", + id: documentId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] }, + }, + ], + [{ type: "entry", value: entry(lastId, rootId) }], + ], + (index, seq) => { + receipts.push([index, seq]); + // Receipts follow the last durable batch: a read started at the first observes it. + read ??= storage.entry(lastId, context); + }, + context, + ); + expect(result).toEqual({ stored: 3 }); + expect(receipts.map(([index]) => index)).toEqual([0, 1, 2]); + const [[, first], [, second], [, last]] = receipts as [[number, Seq], [number, Seq], [number, Seq]]; + expect(second).toBeGreaterThan(first); + expect(last).toBeGreaterThan(second); + expect((await read)?.commitSeq).toBe(last); + expect((await storage.entry(firstId, context))?.commitSeq).toBe(first); + expect((await storage.document(documentId, "current", context))?.value).toEqual({ count: 2 }); + const next = await commitWrites( + storage, + [{ type: "entry", value: entry(await storage.mintId(), rootId) }], + context, + ); + expect(next).toBeGreaterThan(last); + }, + ), + + createCase(options, "stores the batches before the first one refused, and none after it", async (storage) => { + const rootId = await createRoot(storage); + const firstId = await storage.mintId(); + const afterId = await storage.mintId(); + const receipts: number[] = []; + const result = await storage.commit( + [ + [{ type: "entry", value: entry(firstId, rootId) }], + // Refused against the state the first batch left: the ID is taken now. + [ + { type: "task", value: pendingTask(await storage.mintId>(), rootId) }, + { type: "entry", value: entry(firstId, rootId) }, + ], + [{ type: "entry", value: entry(afterId, rootId) }], + ], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: ID ${firstId} already belongs to entry`); + expect(receipts).toEqual([0]); + expect(await storage.entry(firstId, context)).toBeDefined(); + expect(await storage.entry(afterId, context)).toBeUndefined(); + expect((await storage.scanTasks({}, 10, undefined, context)).items.map((task) => task.id)).toEqual([]); + + // A refused first batch stores nothing; the storage goes on. + const refused = await storage.commit( + [[{ type: "conversation", value: { id: rootId } }]], + (index) => receipts.push(index), + context, + ); + expect(refused.stored).toBe(0); + expect(String(refused.refused)).toBe(`Error: ID ${rootId} already belongs to conversation`); + expect(receipts).toEqual([0]); + await commitWrites(storage, [{ type: "entry", value: entry(afterId, rootId) }], context); + expect(await storage.entry(afterId, context)).toBeDefined(); + }), + + createCase( + options, + "refuses a later batch its own malformed actions fail, keeping the batches before it", + async (storage) => { + const rootId = await createRoot(storage); + const documentId = await storage.mintId(); + await commitWrites( + storage, + [ + { + type: "document.create", + record: { id: documentId, kind: "staged", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 1 } }, + }, + ], + context, + ); + const firstId = await storage.mintId(); + const secondId = await storage.mintId(); + const afterId = await storage.mintId(); + const receipts: number[] = []; + const result = await storage.commit( + [ + [{ type: "entry", value: entry(firstId, rootId) }], + // Malformed in itself, whatever is stored: one document retired twice. + [ + { type: "entry", value: entry(secondId, rootId) }, + { type: "document.retire", id: documentId }, + { type: "document.retire", id: documentId }, + ], + [{ type: "entry", value: entry(afterId, rootId) }], + ], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(result.refused).toBeDefined(); + expect(receipts).toEqual([0]); + expect(await storage.entry(firstId, context)).toBeDefined(); + expect(await storage.entry(secondId, context)).toBeUndefined(); + expect(await storage.entry(afterId, context)).toBeUndefined(); + expect((await storage.document(documentId, "current", context))?.value).toEqual({ n: 1 }); + }, + ), + + createCase(options, "resolves a batched copy against the state the earlier batches left", async (storage) => { + const rootId = await createRoot(storage); + const childId = await storage.mintId(); + const sourceId = await storage.mintId(); + const copyId = await storage.mintId(); + const record = (id: DocumentId, conversationId: ConversationId): DocumentCreate => ({ + id, + kind: "batch.copy", + scope: { kind: "conversation", conversationId }, + history: "rewindable", + fork: "asOf", + }); + const seqs: Seq[] = []; + const result = await storage.commit( + [ + [ + { + type: "document.create", + record: record(sourceId, rootId), + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + [ + { + type: "document.change", + id: sourceId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] }, + }, + ], + [ + { type: "conversation", value: { id: childId } }, + { + type: "document.copy", + record: record(copyId, childId), + source: { id: sourceId, at: "current" }, + }, + ], + ], + (_index, seq) => seqs.push(seq), + context, + ); + expect(result).toEqual({ stored: 3 }); + expect((await storage.document(copyId, "current", context))?.value).toEqual({ count: 2 }); + expect((await storage.document(sourceId, seqs[0]!, context))?.value).toEqual({ count: 1 }); + + // A copy whose source the same batch changes is refused before the batch writes anything. + const laterId = await storage.mintId(); + const laterCopyId = await storage.mintId(); + const refused = await storage.commit( + [ + [ + { type: "conversation", value: { id: laterId } }, + { + type: "document.copy", + record: record(laterCopyId, laterId), + source: { id: sourceId, at: "current" }, + }, + { + type: "document.change", + id: sourceId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] }, + }, + ], + ], + () => {}, + context, + ); + expect(refused.stored).toBe(0); + expect(refused.refused).toBeDefined(); + expect(await storage.conversation(laterId, context)).toBeUndefined(); + expect((await storage.document(sourceId, "current", context))?.value).toEqual({ count: 2 }); + }), + createCase(options, "rejects every operation after close", async (storage) => { await createRoot(storage); await storage.close(context); await expect(storage.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("closed"); - await expect(storage.commit([] satisfies StorageWrite[], adopt, context)).rejects.toThrow("closed"); + await expect( + commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context), + ).rejects.toThrow("closed"); await expect(storage.mintId()).rejects.toThrow("closed"); }), ]; diff --git a/packages/durable/src/tool-search/tool-search.ts b/packages/durable/src/tool-search/tool-search.ts index ca600ae5e..5d79b007e 100644 --- a/packages/durable/src/tool-search/tool-search.ts +++ b/packages/durable/src/tool-search/tool-search.ts @@ -3,7 +3,7 @@ * and the matches are loaded with `control.addTools`, so the next request offers them. Loading writes the * conversation's `modelTools` (its `add`), so loaded tools stay offered on that branch, also after a compaction. */ -import { Type } from "@earendil-works/pi-ai"; +import { num, obj, opt, str } from "@earendil-works/pi-schema"; import { defineExtension, defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; import { rankDocuments } from "../search.ts"; @@ -14,12 +14,13 @@ export const DEFAULT_TOOL_SEARCH_LIMIT = 8; /** Fixed, so it does not change while deferred tools are registered, for example as MCP servers connect. */ const TOOL_SEARCH_DESCRIPTION = `# Tool discovery\n\nSearches over deferred tool metadata with BM25 and exposes matching tools for the next model call.\n\nSome of the tools, such as tools of MCP servers, may not have been provided to you upfront, and you should use this tool (\`${TOOL_SEARCH_TOOL_NAME}\`) to search for the required tools. For MCP tool discovery, always use \`${TOOL_SEARCH_TOOL_NAME}\`.`; -const toolSearchSchema = Type.Object({ - query: Type.String({ description: "Search query for deferred tools." }), - limit: Type.Optional( - Type.Number({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` }), - ), -}); +const toolSearchSchema = obj( + { + query: str({ description: "Search query for deferred tools." }), + limit: opt(num({ description: `Maximum number of tools to return. Defaults to ${DEFAULT_TOOL_SEARCH_LIMIT}.` })), + }, + { open: true }, +); /** `tool_search` result details: the tools it loaded. */ export type ToolSearchToolDetails = { loaded: string[] }; @@ -45,9 +46,9 @@ function schemaText(schema: unknown, parts: string[]): void { } } -/** Search text of a tool: its name, the name with `_` as spaces, its description, and its parameters' text. */ +/** Search text of a tool: its description and its parameters' names and descriptions; the ranker reads the name. */ function searchText(tool: ToolRegistration): string { - const parts = [tool.name, tool.name.replaceAll("_", " "), tool.description]; + const parts = [tool.description]; schemaText(tool.parameters, parts); return parts.filter((part) => part.trim() !== "").join(" "); } diff --git a/packages/durable/src/tools/bash.ts b/packages/durable/src/tools/bash.ts index 21d1e1d03..211338f16 100644 --- a/packages/durable/src/tools/bash.ts +++ b/packages/durable/src/tools/bash.ts @@ -1,5 +1,5 @@ import type { Context } from "@earendil-works/chord"; -import { type Static, Type } from "typebox"; +import { bool, num, obj, opt, type Static, str } from "@earendil-works/pi-schema"; import { defineTool } from "../harness/define.ts"; import type { ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts"; @@ -7,23 +7,33 @@ import { requireEnv } from "./env.ts"; const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000; -const bashSchema = Type.Object({ - command: Type.String({ description: "Bash command to execute" }), - timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), -}); +// Open objects, as the tools accepted extra arguments when their schemas were TypeBox's. +const bashSchema = obj( + { + command: str({ description: "Bash command to execute" }), + timeout: opt(num({ description: "Timeout in seconds (optional, no default timeout)" })), + }, + { open: true }, +); -const powershellSchema = Type.Object({ - command: Type.String({ description: "PowerShell command to execute" }), - timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), -}); +const powershellSchema = obj( + { + command: str({ description: "PowerShell command to execute" }), + timeout: opt(num({ description: "Timeout in seconds (optional, no default timeout)" })), + }, + { open: true }, +); /** What a program that calls `bash` or `powershell` receives. */ -const shellStructuredOutputSchema = Type.Object({ - output: Type.String({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), - truncated: Type.Boolean({ description: "Whether earlier output was dropped" }), - fullOutputPath: Type.Optional(Type.String({ description: "File with the complete output, when it was spilled" })), - exitCode: Type.Number(), -}); +const shellStructuredOutputSchema = obj( + { + output: str({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), + truncated: bool({ description: "Whether earlier output was dropped" }), + fullOutputPath: opt(str({ description: "File with the complete output, when it was spilled" })), + exitCode: num(), + }, + { open: true }, +); export type BashToolInput = Static; export type PowerShellToolInput = Static; diff --git a/packages/durable/src/tools/edit.ts b/packages/durable/src/tools/edit.ts index 802150152..e5b9be73d 100644 --- a/packages/durable/src/tools/edit.ts +++ b/packages/durable/src/tools/edit.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { arr, obj, type Static, str } from "@earendil-works/pi-schema"; import type { FileError } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -16,21 +16,27 @@ import { requireEnv } from "./env.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToolPath } from "./path-utils.ts"; -const replaceEditSchema = Type.Object({ - oldText: Type.String({ - description: - "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", - }), - newText: Type.String({ description: "Replacement text for this targeted edit." }), -}); +const replaceEditSchema = obj( + { + oldText: str({ + description: + "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", + }), + newText: str({ description: "Replacement text for this targeted edit." }), + }, + { open: true }, +); -const editSchema = Type.Object({ - path: Type.String({ description: "Path to the file to edit (relative or absolute)" }), - edits: Type.Array(replaceEditSchema, { - description: - "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", - }), -}); +const editSchema = obj( + { + path: str({ description: "Path to the file to edit (relative or absolute)" }), + edits: arr(replaceEditSchema, { + description: + "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", + }), + }, + { open: true }, +); export type EditToolInput = Static; type LegacyEditToolInput = EditToolInput & { oldText?: unknown; newText?: unknown }; diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts index cede96ef3..de84a2b2f 100644 --- a/packages/durable/src/tools/read.ts +++ b/packages/durable/src/tools/read.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { characterEnd } from "@earendil-works/chord/output"; -import { type Static, Type } from "typebox"; +import { lit, num, obj, opt, type Static, str, union } from "@earendil-works/pi-schema"; import { type BinaryReader, type FileInfo, @@ -31,14 +31,26 @@ import { } from "./image-processor.ts"; import { resolveReadToolPath } from "./path-utils.ts"; -const readSchema = Type.Object({ - path: Type.String({ description: "Path to the file to read (relative or absolute)" }), - offset: Type.Optional(Type.Number({ description: "Line number to start reading from (1-indexed)" })), - limit: Type.Optional(Type.Number({ description: "Maximum number of lines to read" })), -}); +const readSchema = obj( + { + path: str({ description: "Path to the file to read (relative or absolute)" }), + offset: opt(num({ description: "Line number to start reading from (1-indexed)" })), + limit: opt(num({ description: "Maximum number of lines to read" })), + }, + { open: true }, +); export type ReadToolInput = Static; +/** + * What programs such as code mode scripts get: the text of a text file, or the image block of an image, which code + * mode's `image()` shows. + */ +const readStructuredOutputSchema = union([ + str(), + obj({ type: lit("image"), data: str(), mimeType: str() }, { open: true }), +]); + export type ReadToolDetails = { /** How the shown text was cut; the text itself is the result content. */ truncation?: Omit; @@ -84,7 +96,7 @@ export type ReadToolOptions = { /** * Reads text files and images. Remarks about truncation, continuation, and image processing are diagnostics; the content - * is only file text, or one image, which programs get as its `ImageContent`. + * is only file text, or one image. Programs get the text, or the image's `ImageContent`. */ export function createReadTool(options: ReadToolOptions = {}): ToolRegistration { const formats = options.images === undefined ? "jpg, png, gif, webp" : "jpg, png, gif, webp, bmp"; @@ -92,6 +104,7 @@ export function createReadTool(options: ReadToolOptions = {}): ToolRegistration< name: "read", description: `Read the contents of a file. Supports text files and images (${formats}). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, parameters: readSchema, + structuredOutputSchema: readStructuredOutputSchema, async execute(args, api, context) { const { path, offset, limit } = args; const env = requireEnv(api); @@ -187,7 +200,8 @@ async function readImage( const diagnostics: ToolDiagnostic[] = [ { severity: "info", code: "image", message: [`Read image file [${image.mimeType}].`, ...notes].join(" ") }, ]; - return { output: [{ type: "image", data: image.data, mimeType: image.mimeType }], diagnostics }; + const block = { type: "image" as const, data: image.data, mimeType: image.mimeType }; + return { output: [block], structuredOutput: block, diagnostics }; } /** What the conversation's model takes: whether images, and within which limits. */ @@ -306,6 +320,7 @@ async function readText( return { output: outputText === "" ? [] : [{ type: "text" as const, text: outputText }], + structuredOutput: outputText, ...(details === undefined ? {} : { details }), diagnostics, }; diff --git a/packages/durable/src/tools/write.ts b/packages/durable/src/tools/write.ts index ee9f724cb..d8c804558 100644 --- a/packages/durable/src/tools/write.ts +++ b/packages/durable/src/tools/write.ts @@ -1,4 +1,4 @@ -import { type Static, Type } from "typebox"; +import { obj, type Static, str } from "@earendil-works/pi-schema"; import { getOrThrow } from "../env/index.ts"; import { defineTool } from "../harness/define.ts"; import type { ToolRegistration } from "../harness/types.ts"; @@ -6,10 +6,13 @@ import { requireEnv } from "./env.ts"; import { withFileMutationQueue } from "./file-mutation-queue.ts"; import { resolveToolPath } from "./path-utils.ts"; -const writeSchema = Type.Object({ - path: Type.String({ description: "Path to the file to write (relative or absolute)" }), - content: Type.String({ description: "Content to write to the file" }), -}); +const writeSchema = obj( + { + path: str({ description: "Path to the file to write (relative or absolute)" }), + content: str({ description: "Content to write to the file" }), + }, + { open: true }, +); export type WriteToolInput = Static; diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 4f17a3711..027832568 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -981,7 +981,8 @@ export type CommitPublication = { /** * Transaction surface of one Session commit callback. - * Table reads and creation results are trusted immutable values and may be shared with internal commit state. + * Table reads and creation results are trusted immutable values and may be shared with internal commit state; the + * Session may freeze them (its caches and context ranges keep them), so mutating one can throw. */ export interface Tx { conversation(id: ConversationId): Promise; @@ -997,6 +998,8 @@ export interface Tx { scanEntries(query: EntryQuery, limit: number, cursor?: Cursor): Promise>; /** Newest visible entry of the conversation that carries a `head`. */ latestHeadMarker(conversationId: ConversationId): Promise<(EntryRecord & { readonly head: EntryId }) | undefined>; + /** ID of the newest entry visible from the conversation; `undefined` when it has none. */ + latestEntryId(conversationId: ConversationId): Promise; scanTasks( query: TaskQuery, limit: number, @@ -1093,16 +1096,18 @@ export type DocumentWatch = WatchHandle>; /** * A synchronous observer of committed Durable state, for adapters such as a Chord source (`observeDoc()`, * `Harness.observeConversation()`, `Harness.observeTaskGraph()`), with the shapes of Chord's watch: `advance` gets a - * `WatchCall`, `end` a `WatchEnd`. `advance` gets every change, unfolded, as memory adopts it: for a stored commit inside - * Storage's `adopt`, for a deferred one before it is stored. A view or the graph gets at most one call per commit, a - * document observer one per change at its address (two for a commit that retires and recreates it); the task graph also - * advances outside any commit and `adopt`, on the Session line, when a registry change or a failed migration changes a + * `WatchCall`, `end` a `WatchEnd`. `advance` gets every change, unfolded, as memory adopts it: for a stored commit at its + * publication once Storage stored it, for a deferred one before it is stored. A view or the graph gets at most one call + * per commit, a document observer one per change at its address (two for a commit that retires and recreates it); the + * task graph also advances outside any commit's publication, on the Session line, when a registry change or a failed + * migration changes a * `blocked` reason. Durable's failures are final, so every call is a `value` call; a failure ends the observer. Calls * may come before the attaching promise resolves; their ops apply from `Observation.value`. `advance` must only record * the call, synchronously and quickly, never await, call the Session, or throw. A throw is reported as * `listener_error`, detaches the observer, and ends it `listener_error` with the error. `end` is called once: `ended` - * when the Session closes or the observed scope's owner ends (a task document's task, after its retirement change, - * inside that commit's `adopt`), `failed` with the error that failed the Session; never after `detach()`. + * when the observed scope's owner ends (a task document's task, after its retirement change, inside that commit's + * `adopt`), `failed` with "session closed" when the Session closes (an adapter follows the next Session) or with the + * error that failed the Session; never after `detach()`. */ export type Observer = { advance(call: WatchCall, context: Context): void; @@ -1111,7 +1116,7 @@ export type Observer = { /** * A synchronous observer of stored commits (`Session.observeCommits()`): `advance` gets each `CommitPublication`, `end` - * the Session's close. + * the Session's close (`failed` with "session closed", or with the error that failed the Session). */ export type CommitObserver = { advance(publication: CommitPublication, context: Context): void; @@ -1187,7 +1192,10 @@ export type CommitOptions = { readonly persistence?: CommitPersistence }; export interface Session extends DocumentObserver { /** * Run one atomic transaction on the Session mutation line. Inside the callback read only through `tx`: `snapshot()`, `watchDoc()`, `getTask()`, and the other Session and runtime reads may queue on the line the callback holds, and then never resolve. - * `options.persistence` may defer storing it (`CommitPersistence`). + * `options.persistence` may defer storing it (`CommitPersistence`). The callback may run while an earlier commit is + * being stored: at its first read or write of what an unpublished commit wrote, it waits until that commit is stored + * and published; code before that runs early, so act on what it computes only once the commit resolves. Resolves + * once the commit is stored and published, after every earlier commit's outcome. */ commit(change: (tx: Tx) => T | Promise, context: Context, options?: CommitOptions): Promise; /** Seal admission, settle admitted commits, then close storage. */ @@ -1242,10 +1250,10 @@ export interface Session extends DocumentObserver { ): Promise> | undefined>; /** * Observe stored commits, each with exactly what it stored, except that a document the commit retires is published - * as its retirement only; a commit held in memory is published when it is stored (`CommitPublication`). Storage's - * adoption of the commit queues one dispatch, which runs a microtask later, before every wait the commit resolves - * and every abort it queues, and calls the observers attached by then; a commit adopted while none was attached has - * no dispatch. There is no order relative to watch listeners. `advance` runs synchronously and must not block or + * as its retirement only; a commit held in memory is published when it is stored (`CommitPublication`). Commits are + * published in line order once Storage stored them, a macrotask apart; a publication queues one dispatch, which runs + * a microtask later, before every wait the commit resolves and every abort it queues, and calls the observers + * attached by then; a commit published while none was attached has no dispatch. There is no order relative to watch listeners. `advance` runs synchronously and must not block or * call Session APIs; a throw is reported as `listener_error`, detaches the observer, and ends it `listener_error`. * `end` is called once, when the Session closes (`ended`, after its last publication) or fails (`failed`), never * after `detach()`. Attaches synchronously; the observation has no `value`: commits are events, not a state, and the @@ -1302,6 +1310,9 @@ export interface Session extends DocumentObserver { ): Promise | undefined>; } +/** What `Storage.commit()` stored: the number of durable batches, and why the next one was refused, if one was. */ +export type CommitResult = { readonly stored: number; readonly refused?: unknown }; + /** * Atomic persistence boundary for Session records. * @@ -1317,13 +1328,29 @@ export interface Session extends DocumentObserver { */ export interface Storage { /** - * Atomically persist one batch and return its sequence. Once the batch is durable, call `adopt(seq)` exactly once, - * synchronously and before this storage's next commit starts, then resolve with `seq`; a commit that fails before it - * is durable never calls `adopt`. `adopt` is the caller's in-memory adoption and publication of the batch, so memory - * and the Session's views always show a complete prefix of the log; it runs the Session's own observers, which call - * no Storage method, and never throws. Once resolved, later reads through this storage observe the batch. + * Persist `batches` in order, each as one ordinary atomic commit with its own seq, checked against the state the + * earlier ones left, in as few round trips as the backend can. Every batch has at least one write: a commit without + * writes never reaches Storage, so the Session never sends an empty batch. Stops at the first batch Durable's own checks refuse + * (global IDs, document actions, copy resolution): the batches before it are durable, it and the ones after are not. + * Calls `receive(index, seq)` once per durable batch, in order, synchronously, after all of them are durable and + * before the next commit starts. Resolves with the number of durable batches and the refusal, if any. Rejects for an + * unknown outcome (connection lost, statement or commit failed); `receive` calls made before the rejection still + * stand. `receive` is a durability receipt, not publication: the Session publishes later. Once it resolved, later + * reads through this storage observe the stored batches. */ - commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; + commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + context: Context, + ): Promise; + + /** + * Optional. Run `publish`, the Session's publication of one stored or held commit (its cache write-through, + * adoption, and observers), inside this storage's own scope, such as a store's attached host batch, so what the + * publication marks lands together. Call `publish` exactly once, now or soon after; the Session publishes the next + * commit only once it ran. Without it, the Session publishes directly. + */ + publishing?(publish: () => void, context: Context): void; /** * Return a fresh branded candidate from the Session-global numeric ID namespace, greater than every ID minted since diff --git a/packages/durable/test/codemode-bench.ts b/packages/durable/test/codemode-bench.ts index c9c3c2b17..d79df3da6 100644 --- a/packages/durable/test/codemode-bench.ts +++ b/packages/durable/test/codemode-bench.ts @@ -5,6 +5,7 @@ * * node --conditions=source --experimental-strip-types test/codemode-bench.ts [N,...] */ + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -12,7 +13,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; -import { Type } from "typebox"; +import { num, obj, str } from "@earendil-works/pi-schema"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { createRegistry, defineExtension, defineTool, Harness } from "../src/index.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -22,14 +23,14 @@ type Shape = "sequential" | "parallel"; const echo = defineTool({ name: "echo", description: "echo", - parameters: Type.Object({ i: Type.Number() }), + parameters: obj({ i: num() }), execute: async (args) => ({ output: [{ type: "text", text: String(args.i) }] }), }); const plain = defineTool({ name: "plain", description: "plain", - parameters: Type.Object({ n: Type.Number(), shape: Type.String() }), + parameters: obj({ n: num(), shape: str() }), replay: "safe", execute: async (args, api, callContext) => { if (args.shape === "sequential") { diff --git a/packages/durable/test/codemode-describe.test.ts b/packages/durable/test/codemode-describe.test.ts index 99ced0465..85dfdbe99 100644 --- a/packages/durable/test/codemode-describe.test.ts +++ b/packages/durable/test/codemode-describe.test.ts @@ -1,7 +1,8 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText } from "@earendil-works/pi-ai"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; import { CODEMODE_SOURCE_GRAMMAR } from "@earendil-works/pi-codemode/source"; import { defineTool, MemoryStorage, type ToolRegistration } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { createCodemodeExtension } from "../src/codemode/codemode.ts"; import { type CodemodeNamespace, codemodeDescription, codemodeSection } from "../src/codemode/describe.ts"; @@ -12,7 +13,7 @@ function tool(name: string, description: string, extra: Partial ({ output: [] }), ...extra, }); diff --git a/packages/durable/test/codemode.test.ts b/packages/durable/test/codemode.test.ts index 460836d72..c12eba6d2 100644 --- a/packages/durable/test/codemode.test.ts +++ b/packages/durable/test/codemode.test.ts @@ -1,8 +1,8 @@ -import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type CodemodeRun, CodemodeSandbox } from "@earendil-works/pi-codemode"; import { defineTool, @@ -13,6 +13,7 @@ import { ToolTask, type ToolTaskInput, } from "@earendil-works/pi-durable"; +import { bool, num, obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { type CodemodeGlobal, @@ -24,12 +25,13 @@ import { import type { CodemodeNamespace } from "../src/codemode/describe.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { createReadTool } from "../src/tools/read.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()), id: Type.Optional(Type.Number()) }); +const Parameters = obj({ text: opt(str()), id: opt(num()) }); function tool( name: string, @@ -714,13 +716,13 @@ describe("durable code mode", () => { it("resolves a schema tool's error result that carries its value, and rejects one without with its text", async () => { const chat = codemodeChat(); - const Exit = Type.Object({ output: Type.String(), exitCode: Type.Number() }); + const Exit = obj({ output: str(), exitCode: num() }); addTool( chat.setup.registry, defineTool({ name: "run", description: "run", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Exit, // Like bash on a nonzero exit: an error result with its value. execute: async () => ({ @@ -735,7 +737,7 @@ describe("durable code mode", () => { defineTool({ name: "denied", description: "denied", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Exit, execute: async () => ({ output: [{ type: "text", text: "permission denied" }], isError: true }), }), @@ -766,7 +768,7 @@ describe("durable code mode", () => { defineTool({ name: "picture", description: "picture", - parameters: Type.Object({ caption: Type.Boolean() }), + parameters: obj({ caption: bool() }), execute: async (args) => ({ output: [ ...(args.caption ? [{ type: "text" as const, text: "a cat" }] : []), @@ -1047,6 +1049,29 @@ describe("durable code mode output and tool search", () => { expect(message.content.some((item) => item.type === "image")).toBe(true); }); + // #11015 + it("resolves the built-in read with an image's block, which image() shows, and with a text file's text", async () => { + const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + const files = await mkdtemp(join(tmpdir(), "pi-codemode-read-")); + directories.add(files); + await writeFile(join(files, "pixel.png"), Buffer.from(png, "base64")); + await writeFile(join(files, "note.txt"), "hello"); + const chat = codemodeChat(); + addTool(chat.setup.registry, createReadTool()); + const { text, message } = await runScript( + `const picture = await tools.read({ path: ${JSON.stringify(join(files, "pixel.png"))} }); + text([picture.type, picture.mimeType, picture.data.length].join(" ")); + image(picture); + text(await tools.read({ path: ${JSON.stringify(join(files, "note.txt"))} }));`, + chat, + ); + expect(message.isError).toBe(false); + expect(text).toContain(`image image/png ${png.length}`); + expect(text).toMatch(/\[Image saved to \S+ \(image\/png, \d+ bytes\)\]/); + expect(text).toContain("hello"); + expect(message.content.some((item) => item.type === "image")).toBe(true); + }); + it("calls a deferred tool the model is not offered, and finds it in ALL_TOOLS", async () => { const chat = codemodeChat(); addTool( @@ -1054,7 +1079,7 @@ describe("durable code mode output and tool search", () => { defineTool({ name: "later", description: "A deferred tool", - parameters: Type.Object({}), + parameters: obj({}), deferred: true, execute: async () => ({ output: [{ type: "text", text: "later ran" }] }), }), diff --git a/packages/durable/test/examples/05-harness.ts b/packages/durable/test/examples/05-harness.ts index ad33c0d0c..e0fd947fb 100644 --- a/packages/durable/test/examples/05-harness.ts +++ b/packages/durable/test/examples/05-harness.ts @@ -1,9 +1,10 @@ // Open a Harness with a registry of extensions. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/05-harness.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { AgentDoc, createRegistry, @@ -23,7 +24,7 @@ const context = BACKGROUND_CONTEXT; const read = defineTool({ name: "read", description: "Read a file", - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `contents of ${args.path}` }] }), }); const Files = defineExtension({ name: "files", tools: [read] }); diff --git a/packages/durable/test/examples/06-configuration.ts b/packages/durable/test/examples/06-configuration.ts index 03184b6d8..fb50401db 100644 --- a/packages/durable/test/examples/06-configuration.ts +++ b/packages/durable/test/examples/06-configuration.ts @@ -1,9 +1,10 @@ // Per-conversation agent choices and Harness-wide settings. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/06-configuration.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { AgentDoc, createRegistry, @@ -25,7 +26,7 @@ function exampleTool(name: string, description: string): AppTool { const tool = defineTool({ name, description, - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); return { ...tool, snippet: `Use ${name} for files.` }; diff --git a/packages/durable/test/examples/09-registry-reload.ts b/packages/durable/test/examples/09-registry-reload.ts index 4135650ee..e1ee8319e 100644 --- a/packages/durable/test/examples/09-registry-reload.ts +++ b/packages/durable/test/examples/09-registry-reload.ts @@ -1,9 +1,10 @@ // Reload extension code through the registry. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/09-registry-reload.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, MemoryStorage, wrapTool } from "../../src/index.ts"; const context = BACKGROUND_CONTEXT; @@ -12,7 +13,7 @@ function exampleTool(name: string, description: string) { return defineTool({ name, description, - parameters: Type.Object({ path: Type.String() }), + parameters: obj({ path: str() }), execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); } diff --git a/packages/durable/test/examples/10-extension-state.ts b/packages/durable/test/examples/10-extension-state.ts index f2d518214..bfd6613e7 100644 --- a/packages/durable/test/examples/10-extension-state.ts +++ b/packages/durable/test/examples/10-extension-state.ts @@ -1,10 +1,11 @@ // An extension that keeps its own per-conversation document. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/10-extension-state.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineDoc, @@ -38,7 +39,7 @@ const Todo = defineExtension({ defineTool({ name: "todo", description: "Add an item to your todo list", - parameters: Type.Object({ item: Type.String() }), + parameters: obj({ item: str() }), execute: async (args, api, callContext) => { await api.commit(async (tx) => { (await tx.doc(Todos, api.conversationId)).items.push(args.item); diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index 73b53076b..59eb2d0d1 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -2,10 +2,11 @@ // view, and then only what changes after that. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/20-late-join.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -27,7 +28,7 @@ registry.install( defineTool({ name: "count", description: "Counts to ten", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { for (let n = 1; n <= 10; n++) { api.output(`${n}\n`); diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index af4a31823..7f624a018 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -4,12 +4,14 @@ // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/21-subagent-foreground.ts + import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type AssistantMessage, type FauxResponseStep, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, FauxResponseStep } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; +import { obj, str } from "@earendil-works/pi-schema"; import { AssistantEntry, type ConversationId, @@ -42,7 +44,7 @@ const Subagent: Extension = defineExtension({ defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String({ description: "What the subagent should do" }) }), + parameters: obj({ task: str({ description: "What the subagent should do" }) }), // Safe to rerun after a crash: a rerun finds the child it already created and the submission it already made. replay: "safe", execute: async (args, api, callContext) => { diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 7aaf64d8b..f01ee89b4 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -5,14 +5,16 @@ // Uses OpenAI when OPENAI_API_KEY is set, and a scripted faux model otherwise. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/22-subagent-background.ts + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type AssistantMessage, type FauxResponseStep, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, FauxResponseStep } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; +import { bool, obj, oneOf, opt, str } from "@earendil-works/pi-schema"; import { AssistantEntry, type ConversationId, @@ -134,11 +136,11 @@ const subagentTool = defineTool({ "Manage persistent subagents that work in the background. Actions: spawn (name, message), send (name, message; " + "followUp: true queues it after the current answer instead of steering), stop (name: aborts its current work), " + "status (name, or all subagents without one). Answers are reported back to you when they arrive.", - parameters: Type.Object({ - action: Type.Union([Type.Literal("spawn"), Type.Literal("send"), Type.Literal("stop"), Type.Literal("status")]), - name: Type.Optional(Type.String()), - message: Type.Optional(Type.String()), - followUp: Type.Optional(Type.Boolean()), + parameters: obj({ + action: oneOf(["spawn", "send", "stop", "status"]), + name: opt(str()), + message: opt(str()), + followUp: opt(bool()), }), // A call interrupted by a crash is not rerun: repeating `stop` could stop newer work. The model sees that the call // was interrupted and can check with `status`. diff --git a/packages/durable/test/examples/26-plan-mode.ts b/packages/durable/test/examples/26-plan-mode.ts index b0bedd056..0eaf5b939 100644 --- a/packages/durable/test/examples/26-plan-mode.ts +++ b/packages/durable/test/examples/26-plan-mode.ts @@ -2,13 +2,14 @@ // extension's own document, and switching back restores the full tool set. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/26-plan-mode.ts + import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { arr, obj, str } from "@earendil-works/pi-schema"; import { NodeExecutionEnv } from "../../src/env/node.ts"; import { createRegistry, @@ -39,7 +40,7 @@ const PlanDoc = defineDoc<{ steps: string[] }>({ const submitPlan = defineTool({ name: "submit_plan", description: "Submit the plan as a list of steps.", - parameters: Type.Object({ steps: Type.Array(Type.String()) }), + parameters: obj({ steps: arr(str()) }), execute: async (args, api, callContext) => { await api.commit(async (tx) => { (await tx.doc(PlanDoc, api.conversationId)).steps = args.steps; diff --git a/packages/durable/test/examples/30-reload-and-restart.ts b/packages/durable/test/examples/30-reload-and-restart.ts index 6813a7659..4918cea2f 100644 --- a/packages/durable/test/examples/30-reload-and-restart.ts +++ b/packages/durable/test/examples/30-reload-and-restart.ts @@ -2,13 +2,15 @@ // with; stored choices are names, so they survive a restart and bind to whatever code the new process installs. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/30-reload-and-restart.ts + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { type ToolResultMessage, Type } from "@earendil-works/pi-ai"; +import type { ToolResultMessage } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, ToolResultEntry } from "../../src/index.ts"; import { openNodeSqliteStorage } from "../../src/storage/sqlite/node.ts"; @@ -24,7 +26,7 @@ function loadVersioned(version: string) { defineTool({ name: "version", description: "Report the tool's code version", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { started(); await gate; diff --git a/packages/durable/test/examples/32-tests-after-edits.ts b/packages/durable/test/examples/32-tests-after-edits.ts index 32292c521..240bfec31 100644 --- a/packages/durable/test/examples/32-tests-after-edits.ts +++ b/packages/durable/test/examples/32-tests-after-edits.ts @@ -2,10 +2,11 @@ // tests, a failure starts a fix run, and input sent meanwhile waits for them too. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/32-tests-after-edits.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -34,7 +35,7 @@ const runTests = () => { const edit = defineTool({ name: "edit", description: "Replace a file's content", - parameters: Type.Object({ path: Type.String(), content: Type.String() }), + parameters: obj({ path: str(), content: str() }), execute: async (args) => { files.set(args.path, args.content); return { output: [{ type: "text", text: `wrote ${args.path}` }] }; diff --git a/packages/durable/test/examples/33-turn-limit-and-continue.ts b/packages/durable/test/examples/33-turn-limit-and-continue.ts index cac9d315c..817bd1fcb 100644 --- a/packages/durable/test/examples/33-turn-limit-and-continue.ts +++ b/packages/durable/test/examples/33-turn-limit-and-continue.ts @@ -2,10 +2,11 @@ // message, with a fresh budget. A host learns that a run stopped from `pi.live.runFailure`. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/33-turn-limit-and-continue.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { num, obj } from "@earendil-works/pi-schema"; import { ConversationBusy, createRegistry, @@ -21,7 +22,7 @@ const context = BACKGROUND_CONTEXT; const step = defineTool({ name: "step", description: "Do one step of the work", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (args) => ({ output: [{ type: "text", text: `step ${args.n} done` }] }), }); const call = (n: number) => fauxAssistantMessage(fauxToolCall("step", { n }), { stopReason: "toolUse" }); diff --git a/packages/durable/test/examples/34-submit-in-commit.ts b/packages/durable/test/examples/34-submit-in-commit.ts index 291305cc6..5cdfa918f 100644 --- a/packages/durable/test/examples/34-submit-in-commit.ts +++ b/packages/durable/test/examples/34-submit-in-commit.ts @@ -3,10 +3,11 @@ // state; `whenIdle: "reject"` refuses it when no run is working. // Run from packages/durable: // node --conditions=source --experimental-strip-types test/examples/34-submit-in-commit.ts + import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { obj } from "@earendil-works/pi-schema"; import { ConversationIdle, createRegistry, @@ -38,7 +39,7 @@ const working = new Promise((resolve) => { const build = defineTool({ name: "build", description: "Build the project", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await working; return { output: [{ type: "text", text: "built" }] }; diff --git a/packages/durable/test/examples/37-harness-registry.ts b/packages/durable/test/examples/37-harness-registry.ts new file mode 100644 index 000000000..404a61a0d --- /dev/null +++ b/packages/durable/test/examples/37-harness-registry.ts @@ -0,0 +1,172 @@ +// A Durable harness in Chord objects, p3's pattern (Chord v2 slice 8): the object that opens Durable on the host's store +// takes its extensions from a Chord registry (`registryReader`), so any plugin extends the harness by contributing an +// extension; and it contributes its Harness to a second registry, through which another object follows its views +// (`self.follow`). When that Harness's Session closes, the follower fails and follows the next Harness, woken by its +// contribution (`self.retryOn`). +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/37-harness-registry.ts + +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + defineRegistry, + local, + method, + type Native, + openStore, + type RegistryTable, + sqliteBackend, + str, +} from "@earendil-works/chord/next"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { createModels } from "@earendil-works/pi-ai"; +import { + createRegistry, + defineExtension, + defineTask, + type Extension, + Harness, + type RegistrySnapshot, + ROOT_CONVERSATION_ID, + registryReader, +} from "../../src/index.ts"; +import { StoreStorage } from "../../src/storage/store/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// 1. The core's registries: extensions in, Durable's registry snapshot out; and the Harness, for other objects. +const Extensions = defineRegistry("example.extensions", { version: 1 }); +const Harnesses = defineRegistry, Harness | undefined>("example.harnesses", { version: 1 }); + +/** Builds Durable's snapshot without throwing: an extension Durable refuses (a task name taken) is reported, left out. */ +function snapshotOf( + table: RegistryTable, + report: (conflict: { message: string; plugins: string[] }) => void, +) { + const registry = createRegistry(); + for (const { value, plugin } of table) { + try { + registry.install(value); + } catch (error) { + report({ message: (error as Error).message, plugins: [plugin] }); + } + } + return registry.snapshot(); +} + +const Core = local("example.harness", { + version: 1, + params: {}, + members: { greet: method(undefined, str()), close: method() }, +}); +const Transcript = local("example.transcript", { version: 1, params: {}, members: { kinds: arr(str()) } }); + +// 2. The core facet: the harness object opens Durable on the store with its extensions read from the registry. +const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); +const core = defineFacet((host) => { + host.serve(Extensions, { compose: (table, { report }) => snapshotOf(table, report) }); + host.serve(Harnesses, { compose: (table) => table.at(-1)?.value.value }); + host.serve(Core, (_, self) => { + // A declared dependency: the object activates (and opens Durable) once the registry has a value. + const extensions = self.registry(Extensions); + const harness = self.open( + async (open) => { + const opened = await Harness.open( + await StoreStorage.open(store, Core.name, open), + { models: createModels(), registry: registryReader(extensions, open) }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + self.contribute(Harnesses, harness); + // Following its own task graph, the object fails with its Session (S1), and its retry opens the next Harness. + self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + return { + greet: async (ctx) => { + // The composed snapshot, read at use: is the task's extension installed? + if (extensions.value.task(Greet.definition.name) === undefined) return "no greeter installed"; + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + const id = await root!.commit( + (tx) => tx.createTask(Greet, undefined, { ownership: { kind: "conversation" } }), + ctx, + ); + harness.value.resume(); + return JSON.stringify((await harness.value.waitForTask(id, ctx)).state.outcome); + }, + close: async (ctx) => { + await harness.value.close(ctx); + return null; + }, + }; + }); + // 3. Another object follows the harness's conversation view through the registry. + host.serve(Transcript, (_, self) => { + const harness = self.registry(Harnesses); + self.retryOn(harness); + const view = self.follow(async (observer, ctx) => + harness.value?.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + return { kinds: self.derive([view], ([v]) => v.entries.map((entry) => entry.kind)) }; + }); +}); + +// 4. Two plugins contribute extensions with the same task name, which Durable refuses: the first by plugin name +// (copycat) is installed, the other reported. +const Greet = defineTask({ + name: "example.greet", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + await runtime.commit(async (tx) => { + await tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "example.greeted", data: {} }); + return { status: "terminal", outcome: { status: "completed", result: "hello" } }; + }, taskContext); + }, + }, + abort: async () => {}, +}); +const greeter = defineFacet((_, self) => + self.contribute(Extensions, defineExtension({ name: "greeter", tasks: [Greet] })), +); +const copycat = defineFacet((_, self) => + self.contribute(Extensions, defineExtension({ name: "copycat", tasks: [Greet] })), +); + +const host = createHost({ + store, + onError: (error) => console.log("onError:", error.kind, error.kind === "registry_conflict" ? error.message : ""), +}); +await host.load( + [ + { name: "@example/core", facet: core }, + { name: "@example/greeter", facet: greeter }, + { name: "@example/copycat", facet: copycat }, + ], + context, +); + +// 5. The task comes from a contributed extension; the transcript object sees its entry. (It failed once at start, before +// the harness object contributed its Harness: `activation_failed`, then the contribution woke it.) +const harness = host.hold(Core, {}, context); +const transcript = host.hold(Transcript, {}, context); +console.log("greet:", await harness.greet(context)); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("transcript:", transcript.kinds.value); + +// 6. The Session closes: both objects fail; a retry of the harness object opens the next Harness (on the same store: +// the transcript is still there), whose contribution wakes the follower. +await harness.close(context).catch((error: Error) => console.log("close:", error.message)); +await new Promise((resolve) => setTimeout(resolve, 10)); +console.log("transcript after the close:", transcript.state.value.kind); +harness.retry(); +await new Promise((resolve) => setTimeout(resolve, 50)); +console.log("transcript after the retry:", transcript.state.value.kind, transcript.kinds.value); + +await host.close(context); +await store.close(context); diff --git a/packages/durable/test/harness-after-run.test.ts b/packages/durable/test/harness-after-run.test.ts index 373922122..7201fe6af 100644 --- a/packages/durable/test/harness-after-run.test.ts +++ b/packages/durable/test/harness-after-run.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Draft, JsonValue } from "@earendil-works/chord"; -import { fauxAssistantMessage, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { CompactionTask, ConversationBusy, @@ -25,6 +25,7 @@ import { type TaskId, type Tx, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { endRun } from "../src/harness/live.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -132,7 +133,7 @@ function install(setup: ChatSetup, state: Probe, afterRun?: GenerationHooks["aft return Tests; } -const Parameters = Type.Object({}); +const Parameters = obj({}); const tools = defineExtension({ name: "tools", tools: [ diff --git a/packages/durable/test/harness-blocked.test.ts b/packages/durable/test/harness-blocked.test.ts index bc85d0e52..b51231103 100644 --- a/packages/durable/test/harness-blocked.test.ts +++ b/packages/durable/test/harness-blocked.test.ts @@ -12,6 +12,7 @@ import { } from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { context, flush } from "./session-support.ts"; import { completed, deferred, eventually, openTasks } from "./task-support.ts"; @@ -104,7 +105,7 @@ describe("recovery at open", () => { ...rest, state: { status: "completing", outcome: { status: "completed", result: null } }, } as const; - await raw.commit([{ type: "task", value: completing }], () => {}, context); + await commitWrites(raw, [{ type: "task", value: completing }], context); await raw.close(context); // That pass's commit fails, as on a platform reset. diff --git a/packages/durable/test/harness-callers.test.ts b/packages/durable/test/harness-callers.test.ts index ff9a8311e..bbf4e7db9 100644 --- a/packages/durable/test/harness-callers.test.ts +++ b/packages/durable/test/harness-callers.test.ts @@ -1,4 +1,4 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, @@ -7,6 +7,7 @@ import { type NestedToolExecutionResult, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -16,7 +17,7 @@ function tool(name: string, extra: Partial = {}): ToolRegistra return defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); @@ -34,7 +35,7 @@ function prober(names: readonly string[]) { const registration = defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { for (const name of names) received.set(name, await api.executeTool(name, {}, callContext)); return {}; diff --git a/packages/durable/test/harness-cancellation-barrier.test.ts b/packages/durable/test/harness-cancellation-barrier.test.ts index 086787e43..9564e39b0 100644 --- a/packages/durable/test/harness-cancellation-barrier.test.ts +++ b/packages/durable/test/harness-cancellation-barrier.test.ts @@ -1,5 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { + type CommitResult, createSession, defineTask, MemoryStorage, @@ -8,6 +9,7 @@ import { type TaskId, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { context, flush } from "./session-support.ts"; import { aborted, deferred, eventually, openTasks } from "./task-support.ts"; @@ -26,19 +28,25 @@ const Dormant = defineTask({ }, }); -/** Holds every commit that abort-marks `target`, while set, until released. */ +/** Holds every commit call that abort-marks `target`, while set, until released. */ class HoldMarks extends MemoryStorage { target: TaskId | undefined; readonly held = deferred(); readonly release = deferred(); - override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) { + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { if ( - writes.some((write) => write.type === "task" && write.value.id === this.target && write.value.abortRequested) + batches + .flat() + .some((write) => write.type === "task" && write.value.id === this.target && write.value.abortRequested) ) { this.held.resolve(); await this.release.promise; } - return super.commit(writes, adopt, commitContext); + return super.commit(batches, receive, commitContext); } } @@ -127,7 +135,8 @@ describe("work below a cancelled owner", () => { policy: "failFast", } as const; const failure = { status: "failed", error: { message: "failed" } } as const; - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: { ...owner, abortRequested: true, abortReason: "restart", state: waiting } }, { @@ -135,7 +144,6 @@ describe("work below a cancelled owner", () => { value: { ...failedRecord, state: { status: "terminal", outcome: failure } } as typeof failedRecord, }, ], - () => {}, context, ); // The child's definition is missing: a restart mark would keep it waiting, a request orphans it. @@ -217,9 +225,9 @@ describe("work below a cancelled owner", () => { ); const record = (await storage.task(holder, context))!; const completing = { status: "completing", outcome: { status: "completed", result: null } } as const; - await storage.commit( + await commitWrites( + storage, [{ type: "task", value: { ...record, state: completing } as typeof record }], - () => {}, context, ); const { harness } = await openTasks(storage, [Dormant]); diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index b60eb2dae..1be32ae35 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -9,7 +9,6 @@ import { type Message, type SimpleStreamOptions, type TranscriptContext, - Type, type UserMessage, } from "@earendil-works/pi-ai"; import { @@ -33,6 +32,7 @@ import { type TaskId, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { selectCut, serializeConversation } from "../src/harness/compaction.ts"; import { orderToolResults } from "../src/harness/context.ts"; @@ -471,7 +471,7 @@ describe("manual compaction", () => { defineTool({ name: "wait", description: "wait", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [{ type: "text", text: "waited" }] }), }), ); @@ -886,7 +886,7 @@ describe("compaction outcomes", () => { defineTool({ name: "read", description: "read", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1343,7 +1343,7 @@ describe("compaction estimates and interactions", () => { defineTool({ name: "slow", description: "slow", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { toolReached.resolve(); await toolGate.promise; diff --git a/packages/durable/test/harness-context-range.test.ts b/packages/durable/test/harness-context-range.test.ts new file mode 100644 index 000000000..d44cab3db --- /dev/null +++ b/packages/durable/test/harness-context-range.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { advanceRange, readContext, readContextFrom } from "../src/harness/context.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { MemoryStorage } from "../src/storage/memory.ts"; +import { context } from "./session-support.ts"; + +describe("kept context ranges", () => { + it("is not advanced when an entry was committed while its scan ran off the line", async () => { + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + class PausedStorage extends MemoryStorage { + pause = false; + override async scanEntries(...args: Parameters) { + const page = await super.scanEntries(...args); + if (this.pause) { + this.pause = false; + entered.resolve(); + await release.promise; + } + return page; + } + } + const storage = new PausedStorage(); + const session = new SessionImpl(storage); + const conversation = await session.commit( + (tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), + context, + ); + await session.commit((tx) => tx.appendEntry(conversation.id, { kind: "first" }), context); + storage.pause = true; + const read = readContextFrom(session, storage, conversation.id, context, undefined, undefined); + await entered.promise; + await session.commit((tx) => tx.appendEntry(conversation.id, { kind: "missed" }), context); + release.resolve(); + const { range } = await read; + expect(range?.current).toBe(false); + const last = await session.commit((tx) => tx.appendEntry(conversation.id, { kind: "last" }), context); + // Not current: left as it is, so the next read extends it from Storage instead of skipping "missed". + expect(advanceRange(range!, [last])).toBe(range); + const again = await readContextFrom(session, storage, conversation.id, context, undefined, range); + const fresh = await readContext(session, storage, conversation.id, context); + expect(again.view.entries.map((entry) => entry.kind)).toEqual(["first", "missed", "last"]); + expect(again.view).toEqual(fresh); + expect(again.range?.current).toBe(true); + await session.close(context); + }); +}); diff --git a/packages/durable/test/harness-context.test.ts b/packages/durable/test/harness-context.test.ts index f3a9ada0e..c0e6d13b5 100644 --- a/packages/durable/test/harness-context.test.ts +++ b/packages/durable/test/harness-context.test.ts @@ -444,18 +444,18 @@ describe("conversation context", () => { const initial = await read(); expect(initial.view).toEqual(await root.context(taskContext)); - // Three new entries: the bounds probe reads one row, the extension the three new ones. + // Three new entries: the kept range took them as they were committed, so the read scans nothing. await write({ kind: "message", model: [user("new")] }); await write({ kind: "note", data: { text: "display only" } }); await write({ kind: "message", model: [assistant("answer")] }); const extended = await read(); - expect(extended.rows).toBe(4); + expect(extended.rows).toBe(0); expect(extended.view).toEqual(await root.context(taskContext)); // A newer edit of an older entry applies to the extended range. await write({ kind: "edit", edits: [{ target: first.id, action: "omit" }] }); const edited = await read(); - expect(edited.rows).toBe(2); + expect(edited.rows).toBe(0); expect(edited.view).toEqual(await root.context(taskContext)); expect(edited.view.messages.map(describeMessage)).not.toContain("user:first"); @@ -559,9 +559,9 @@ describe("conversation context", () => { await probe("idle"); clock += 600_000; await probe("expired"); - // Bounds probes read one row each. The child scans its own new entry; the waiting parent's later invocation and - // the probe within the retention period scan nothing new. After it, the probe reads the whole transcript again. - expect(rows).toEqual({ parent: 22, child: 2, after: 1, idle: 1, expired: 23 }); + // The Session knows the bounds, and the kept range takes each committed entry, so only a first read scans: the + // parent's, and the probe's after the retention period, which reads the whole transcript again. + expect(rows).toEqual({ parent: 21, child: 0, after: 0, idle: 0, expired: 22 }); await harness.close(context); }); @@ -732,8 +732,8 @@ describe("conversation context", () => { expect((await read(root)).messages[0]).toBe("user:first"); await append(root.id, { kind: "pi.system", model: [system({ plan: "Plan." })] }); let read2 = await read(root); - // One bounds probe and the added entry: the range was extended, not derived again. - expect(read2.rows).toBe(2); + // Nothing: the kept range took the added entry as it was committed. + expect(read2.rows).toBe(0); expect(read2.messages.slice(0, 3)).toEqual(["system:plan", "user:first", "assistant:old 0"]); // A system message the cached range already has keeps a later first `pi.system` entry in place. @@ -743,7 +743,7 @@ describe("conversation context", () => { expect((await read(fork)).messages).toEqual(["user:first", "assistant:answer", "system:note"]); await append(fork.id, { kind: "pi.system", model: [system({ note: null })] }); read2 = await read(fork); - expect(read2.rows).toBe(2); + expect(read2.rows).toBe(0); expect(read2.messages).toEqual(["user:first", "assistant:answer", "system:note", "system:note"]); // An omitted system message still rules out a later baseline; the extended read keeps that. @@ -761,7 +761,7 @@ describe("conversation context", () => { expect((await read(omitted)).messages).toEqual(["user:first", "assistant:answer"]); await append(omitted.id, { kind: "pi.system", model: [system({ legacy: null })] }); read2 = await read(omitted); - expect(read2.rows).toBe(2); + expect(read2.rows).toBe(0); expect(read2.messages).toEqual(["user:first", "assistant:answer", "system:legacy"]); // A first `pi.system` entry without a system message leaves no baseline for a later one to take. @@ -771,7 +771,7 @@ describe("conversation context", () => { expect((await read(empty)).messages).toEqual(["user:first", "assistant:answer"]); await append(empty.id, { kind: "pi.system", model: [system({ s: "Delta." })] }); read2 = await read(empty); - expect(read2.rows).toBe(2); + expect(read2.rows).toBe(0); expect(read2.messages).toEqual(["user:first", "assistant:answer", "system:s"]); await runtime.commit(() => DONE, taskContext); }, @@ -907,7 +907,7 @@ describe("conversation context", () => { await late; probeStart.resolve(); expect((await harness.waitForTask(probeId, context)).state.outcome).toEqual(DONE.outcome); - expect(rows).toEqual({ probe: 22 }); + expect(rows).toEqual({ probe: 21 }); await harness.close(context); }); @@ -934,7 +934,7 @@ describe("conversation context", () => { vi.advanceTimersByTime(1_000); expect(vi.getTimerCount()).toBe(idle); await probe("second"); - expect(rows).toEqual({ first: 22, second: 22 }); + expect(rows).toEqual({ first: 21, second: 21 }); await harness.close(context); }); @@ -954,7 +954,7 @@ describe("conversation context", () => { expect((await harness.waitForTask(id, context)).state.outcome).toEqual(DONE.outcome); await root.waitForIdle(context); } - expect(rows).toEqual({ first: 22, second: 22 }); + expect(rows).toEqual({ first: 21, second: 21 }); await harness.close(context); }); }); diff --git a/packages/durable/test/harness-continue.test.ts b/packages/durable/test/harness-continue.test.ts index 8b609b8eb..627e6218f 100644 --- a/packages/durable/test/harness-continue.test.ts +++ b/packages/durable/test/harness-continue.test.ts @@ -1,4 +1,5 @@ // `submit({ type: "continue" })`: a run without a new user message (#11043). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -10,7 +11,6 @@ import { fauxText, fauxToolCall, type Message, - Type, } from "@earendil-works/pi-ai"; import { AssistantEntry, @@ -26,6 +26,7 @@ import { type Storage, UserEntry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -125,7 +126,7 @@ describe("continue", () => { defineTool({ name: "wait", description: "Waits until aborted", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, _api, callContext) => { started.resolve(); return aborted(callContext.abortSignal!); @@ -312,7 +313,7 @@ describe("continue", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); diff --git a/packages/durable/test/harness-deferred-progress.test.ts b/packages/durable/test/harness-deferred-progress.test.ts index da77f6f36..07116cc08 100644 --- a/packages/durable/test/harness-deferred-progress.test.ts +++ b/packages/durable/test/harness-deferred-progress.test.ts @@ -2,8 +2,9 @@ // the next commit that reads or writes `pi.live` or within `settings.progress.persistWithinMs`. Crashes are simulated // by failing the Harness: from that moment nothing more reaches Storage, as when the process dies; the reopened Harness // resumes from what was stored. + import { applyImmutable } from "@earendil-works/chord/delta"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AssistantEntry, type ConversationId, @@ -15,6 +16,7 @@ import { type LiveState, ToolResultEntry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import type { PartialMessage } from "../src/harness/partial-changes.ts"; import type { SessionImpl } from "../src/session/session.ts"; @@ -161,7 +163,8 @@ describe("generation partials", () => { const submission = await opened.root.submit({ type: "input", content: "hi" }, context); await waitFor(() => (opened.memory.at(-1)?.length ?? 0) > 40); await crash(opened.harness); - expect(await storedLive(storage, opened.root.id)).toMatchObject({ generation: { attempt: 1 } }); + // The attempt is presentation, deferred like the partials: none of it was stored. + expect((await storedLive(storage, opened.root.id))?.generation).toBeUndefined(); expect(storedText(await storedLive(storage, opened.root.id))).toBeUndefined(); setup.faux.setResponses([fauxAssistantMessage("answer")]); @@ -256,7 +259,7 @@ describe("tool output", () => { const registration = defineTool({ name: "work", description: "work", - parameters: Type.Object({}), + parameters: obj({}), replay, execute: async (_args, api, callContext) => { const run = ++state.runs; @@ -305,7 +308,7 @@ describe("tool output", () => { defineTool({ name: "work", description: "work", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, callContext) => { runs++; diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index a835aea34..7a46a4a9f 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -119,6 +119,8 @@ describe("generation", () => { it("stores partials as deltas and a complete base once nothing is in flight", async () => { const setup = chatSetup({ tokensPerSecond: 200, tokenSize: { min: 1, max: 1 } }); setup.faux.setResponses([fauxAssistantMessage("w".repeat(200))]); + // Store deferred partials well within the stream, so some reach Storage before the answer. + setup.settings.progress = { persistWithinMs: 100 }; const storage = new ControlledStorage(); const { harness, root } = await openChat(storage, setup); const record = await storage.findDocument( diff --git a/packages/durable/test/harness-group-commit.test.ts b/packages/durable/test/harness-group-commit.test.ts new file mode 100644 index 000000000..46563015c --- /dev/null +++ b/packages/durable/test/harness-group-commit.test.ts @@ -0,0 +1,796 @@ +// Group commit and the scheduler (docs/group-commit.md §6): decisions read the scheduler's memory in regions that +// wait for the unpublished commits whose keys they read, memberships written at admission, outcomes, and close. +import type { JsonValue } from "@earendil-works/chord"; +import { + type ConversationId, + createRegistry, + defineExtension, + defineTask, + type Harness, + type TaskId, + type TaskRecord, + type TaskRuntime, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { SCHEDULER_INDEX_SIZES } from "../src/harness/harness.ts"; +import { ownedKey, ROOTS_KEY } from "../src/session/keys.ts"; +import type { SessionImpl } from "../src/session/session.ts"; +import type { Transaction } from "../src/session/transaction.ts"; +import { ControlledStorage, context, flush, followCommits } from "./session-support.ts"; +import { aborted, type Deferred, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +const gates = new Map>(); +function gate(name: string): Deferred { + let found = gates.get(name); + if (found === undefined) { + found = deferred(); + gates.set(name, found); + } + return found; +} + +/** How often each named phase ran. */ +const runs = new Map(); +function ran(name: string): number { + return runs.get(name) ?? 0; +} +function count(name: string): void { + runs.set(name, ran(name) + 1); +} + +afterEach(() => { + gates.clear(); + runs.clear(); +}); + +const COMPLETED = { status: "terminal", outcome: { status: "completed", result: null } } as const; + +function completedWith(result: R) { + return { status: "terminal", outcome: { status: "completed", result } } as const; +} + +/** Holds until its named gate opens, then completes; its abort handler ends it `aborted`. */ +const Hold = defineTask<{ name: string }, { phase: "hold" }, null>({ + name: "gc.hold", + version: 1, + initial: () => ({ phase: "hold" }), + phases: { + hold: async (task, runtime, ctx) => { + count(task.input.name); + await Promise.race([gate(task.input.name).promise, aborted(runtime.signal)]); + await runtime.commit(() => COMPLETED, ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); + +/** Commits its next phase, returns once `.return` opens; the next phase counts its run and completes. */ +const TwoPhases = defineTask<{ name: string }, { phase: "one" } | { phase: "two" }, null>({ + name: "gc.two-phases", + version: 1, + initial: () => ({ phase: "one" }), + phases: { + one: async (task, runtime, ctx) => { + await runtime.commit(() => ({ status: "running", checkpoint: { phase: "two" } }), ctx); + await gate(`${task.input.name}.return`).promise; + }, + two: async (task, runtime, ctx) => { + count(`${task.input.name}.two`); + await runtime.commit(() => COMPLETED, ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); + +/** Once its gate opens, waits on the task in its input; records why a refused wait was refused, then completes. */ +const Waits = defineTask<{ name: string; on?: TaskId }, { phase: "wait" } | { phase: "done" }, null>({ + name: "gc.waits", + version: 1, + initial: () => ({ phase: "wait" }), + phases: { + wait: async (task, runtime, ctx) => { + await gate(task.input.name).promise; + const on = await runtime.memo("on", ctx); + try { + await runtime.commit( + () => ({ status: "waiting", checkpoint: { phase: "done" }, on: [on as TaskId], policy: "allSettled" }), + ctx, + ); + } catch (error) { + refusals.set(task.input.name, error); + await runtime.commit(() => COMPLETED, ctx); + } + }, + done: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx), + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); +const refusals = new Map(); + +async function status(harness: Harness, id: TaskId): Promise { + return (await harness.getTask(id, context))?.state.status; +} + +/** Stage a task record as is, the way only the scheduler and recovery write them. */ +function setTask(tx: unknown, record: TaskRecord): void { + (tx as Transaction).setTask(record); +} + +describe("group commit: scheduler regions", () => { + it("holds a live owner whose end races a creation below its ended tasks: live P, ended B and A, empty C", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); + const b = await tx.createTask(Hold, { name: "b" }, { ownership: { kind: "task", taskId: p } }); + const a = await tx.createTask(Hold, { name: "a" }, { ownership: { kind: "task", taskId: b } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: a } }); + return { p, b, a, c: c.id }; + }, context); + gate("a").resolve(); + await harness.waitForTask(tree.a, context); + gate("b").resolve(); + await harness.waitForTask(tree.b, context); + // The creation of D in C is admitted and in flight when P commits its end. + const held = storage.holdCommits(); + const c = (await harness.conversation(tree.c, context))!; + const creating = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + await held.entered; + // P's decision reads what the creation writes, so it waits for its publication, holding the line. + gate("p").resolve(); + await flush(); + held.release(); + const d = await creating; + await eventually(async () => (await status(harness, tree.p)) === "completing"); + gate("d").resolve(); + await harness.waitForTask(d, context); + expect((await harness.waitForTask(tree.p, context)).state.outcome.status).toBe("completed"); + await harness.close(context); + }); + + it("holds an owner whose end follows a creation below its child's pending end: live P, ending B, conversation C", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); + const b = await tx.createTask(Hold, { name: "b" }, { ownership: { kind: "task", taskId: p } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: b } }); + return { p, b, c: c.id }; + }, context); + const c = (await harness.conversation(tree.c, context))!; + await eventually(async () => (await status(harness, tree.b)) === "running"); + // A call in flight, so what follows is admitted and stays unpublished, in this order. + const held = storage.holdCommits(); + const blocker = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + // B ends; then D is created in C while B still shows live; then P ends. + gate("b").resolve(); + await flush(); + const creating = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + await flush(); + gate("p").resolve(); + await flush(); + held.release(); + await blocker; + const d = await creating; + // In line order D's creation precedes P's decision: P holds while D is live. + await eventually(async () => (await status(harness, tree.p)) === "completing"); + expect(await status(harness, d)).toBe("running"); + gate("d").resolve(); + expect((await harness.waitForTask(tree.p, context)).state.outcome.status).toBe("completed"); + await harness.close(context); + }); + + it("ends a child before its next phase when its owner's abort mark is unpublished at its step", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold, TwoPhases]); + const root = await harness.root(context); + harness.resume(); + const { owner, child } = await root.commit(async (tx) => { + const owner = await tx.createTask(Hold, { name: "o" }, { ownership: { kind: "conversation" } }); + const child = await tx.createTask(TwoPhases, { name: "t" }, { ownership: { kind: "task", taskId: owner } }); + return { owner, child }; + }, context); + await eventually(async () => { + const record = await harness.getTask(child, context); + return record?.state.status === "running" && (record.state.checkpoint as { phase: string }).phase === "two"; + }); + const held = storage.holdCommits(); + const marking = harness.abortTask(owner, context); + await held.entered; + // The child's phase returns: its step reads the intent set, which the unpublished mark changes. + gate("t.return").resolve(); + await flush(); + expect(ran("t.two")).toBe(0); + held.release(); + expect(await marking).toBe("marked"); + expect((await harness.waitForTask(child, context)).state.outcome.status).toBe("aborted"); + expect((await harness.waitForTask(owner, context)).state.outcome.status).toBe("aborted"); + expect(ran("t.two")).toBe(0); + await harness.close(context); + }); + + it("refuses the second of two opposite waits admitted behind each other", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Waits]); + const root = await harness.root(context); + const { x, y } = await root.commit(async (tx) => { + const x = await tx.createTask(Waits, { name: "x" }, { ownership: { kind: "conversation" } }); + const y = await tx.createTask(Waits, { name: "y" }, { ownership: { kind: "conversation" } }); + return { x, y }; + }, context); + // Each waits on the other, named in a memo. + await root.commit(async (tx) => { + // Every table read before the first write. + const records = [(await tx.task(x))!, (await tx.task(y))!]; + setTask(tx, { ...records[0]!, memos: { on: y } } as TaskRecord); + setTask(tx, { ...records[1]!, memos: { on: x } } as TaskRecord); + }, context); + harness.resume(); + await eventually( + async () => (await status(harness, x)) === "running" && (await status(harness, y)) === "running", + ); + const held = storage.holdCommits(); + gate("x").resolve(); + await held.entered; + gate("y").resolve(); + await flush(); + held.release(); + await harness.waitForTask(y, context); + expect(String(refusals.get("y"))).toContain("wait cycle"); + expect(refusals.has("x")).toBe(false); + expect((await harness.waitForTask(x, context)).state.outcome.status).toBe("completed"); + await harness.close(context); + }); + + it("starts no invocation on a reservation before the reservation is published", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Hold, { name: "r" }, { ownership: { kind: "conversation" } }), + context, + ); + // The reservation's write is in flight: a scheduler commit with writes is never delivered before it is published. + const held = storage.holdCommits(); + harness.resume(); + await held.entered; + await flush(); + expect(ran("r")).toBe(0); + held.release(); + await eventually(() => ran("r") === 1); + expect(await status(harness, id)).toBe("running"); + gate("r").resolve(); + await harness.waitForTask(id, context); + await harness.close(context); + }); + + it("runs a migration once although its reservation waits for an unpublished write of the task", async () => { + const storage = new ControlledStorage(); + let migrations = 0; + const V2 = defineTask<{ name: string }, { phase: "hold" }, null>({ + name: "gc.migrated", + version: 2, + initial: () => ({ phase: "hold" }), + migrate: (input, checkpoint) => { + migrations++; + return { input: input as { name: string }, checkpoint: checkpoint as { phase: "hold" } }; + }, + phases: { hold: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx) }, + abort: async () => {}, + }); + const registry = createRegistry(); + registry.install(defineExtension({ name: "v2", tasks: [V2] })); + const { harness } = await openTasks(storage, [], { registry }); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(V2, { name: "m" }, { ownership: { kind: "conversation" } }), + context, + ); + // Stored at version 1, and written again by a commit that is in flight when the reservation reads it. + const held = storage.holdCommits(); + const writing = root.commit(async (tx) => { + const record = (await tx.task(id))!; + setTask(tx, { ...record, version: 1, memos: { touched: true } } as TaskRecord< + JsonValue, + JsonValue, + JsonValue + >); + }, context); + await held.entered; + harness.resume(); + await flush(); + expect(migrations).toBe(0); + held.release(); + await writing; + expect((await harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); + expect(migrations).toBe(1); + await harness.close(context); + }); + + it("reports a failed migration once although its reservation waits for an unpublished write of the task", async () => { + const storage = new ControlledStorage(); + let migrations = 0; + const V2 = defineTask<{ name: string }, { phase: "hold" }, null>({ + name: "gc.unmigratable", + version: 2, + initial: () => ({ phase: "hold" }), + migrate: () => { + migrations++; + throw new Error("cannot migrate"); + }, + phases: { hold: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx) }, + abort: async () => {}, + }); + const registry = createRegistry(); + registry.install(defineExtension({ name: "v2", tasks: [V2] })); + const { harness, infos } = await openTasks(storage, [], { registry }); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(V2, { name: "m" }, { ownership: { kind: "conversation" } }), + context, + ); + const held = storage.holdCommits(); + const writing = root.commit(async (tx) => { + const record = (await tx.task(id))!; + setTask(tx, { ...record, version: 1 } as TaskRecord); + }, context); + await held.entered; + harness.resume(); + await flush(); + held.release(); + await writing; + await eventually(() => migrations > 0); + await flush(); + expect(migrations).toBe(1); + expect(infos.filter((info) => info.kind === "migration_failed")).toHaveLength(1); + await harness.close(context); + }); + + it("makes a reader of the roots wait for a creation under a root a pending end would unlist", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + // Root R is listed for its last live task, T. + const last = await root.commit( + (tx) => tx.createTask(Hold, { name: "t" }, { ownership: { kind: "conversation" } }), + context, + ); + await eventually(async () => (await status(harness, last)) === "running"); + const published = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) if (change.type === "task") published.add(change.value.id); + }); + // T's end is in flight; a creation under R is admitted after it, while R still shows listed. + const held = storage.holdCommits(); + gate("t").resolve(); + await held.entered; + let created: TaskId | undefined; + const creating = root.commit(async (tx) => { + created = await tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }); + return created; + }, context); + await flush(); + // A reader of the roots on the line (global idle's key) waits until the creation is published too. + const session = harness as unknown as SessionImpl; + const reading = session.readOnLine(async () => { + const gate = session.touch(ROOTS_KEY); + if (gate !== undefined) await gate; + return created !== undefined && published.has(created); + }); + held.release(); + await creating; + expect(await reading).toBe(true); + gate("d").resolve(); + await harness.waitForIdle(context); + await harness.close(context); + }); + + it("walks a creation's memberships past an ancestor whose pending end is unpublished", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + // Live P, in the root conversation, owns the empty conversation C. + const { p, c } = await root.commit(async (tx) => { + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: p } }); + return { p, c: c.id }; + }, context); + await eventually(async () => (await status(harness, p)) === "running"); + const published = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) if (change.type === "task") published.add(change.value.id); + }); + // P's end is in flight; a creation in C is admitted after it. Its chain must not stop at P, which shows live. + const conversation = (await harness.conversation(c, context))!; + const held = storage.holdCommits(); + gate("p").resolve(); + await held.entered; + let created: TaskId | undefined; + const creating = conversation.commit(async (tx) => { + created = await tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }); + return created; + }, context); + await flush(); + const session = harness as unknown as SessionImpl; + const reading = session.readOnLine(async () => { + const gate = session.touch(ROOTS_KEY); + if (gate !== undefined) await gate; + return created !== undefined && published.has(created); + }); + held.release(); + await creating; + expect(await reading).toBe(true); + gate("d").resolve(); + await harness.waitForIdle(context); + await harness.close(context); + }); + + it("reserves the live work of the owner above an ancestor whose end is pending, for a creation below it", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + harness.resume(); + // Live G owns P, which owns the empty conversation C. + const { g, p, c } = await root.commit(async (tx) => { + const g = await tx.createTask(Hold, { name: "g" }, { ownership: { kind: "conversation" } }); + const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "task", taskId: g } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: p } }); + return { g, p, c: c.id }; + }, context); + await eventually(async () => (await status(harness, p)) === "running"); + const published = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) if (change.type === "task") published.add(change.value.id); + }); + // P's end is in flight; a creation in C is admitted after it, while P still shows live. + const conversation = (await harness.conversation(c, context))!; + const held = storage.holdCommits(); + gate("p").resolve(); + await held.entered; + let created: TaskId | undefined; + const creating = conversation.commit(async (tx) => { + created = await tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }); + return created; + }, context); + await flush(); + // A reader of G's live work (as G's end decision is) waits until the creation is published too. + const session = harness as unknown as SessionImpl; + const reading = session.readOnLine(async () => { + const gate = session.touch(ownedKey(g)); + if (gate !== undefined) await gate; + return created !== undefined && published.has(created); + }); + held.release(); + await creating; + expect(await reading).toBe(true); + gate("d").resolve(); + gate("g").resolve(); + await harness.waitForTask(g, context); + await harness.close(context); + }); + + it("keeps the nodes a chain load installed for an unpublished creation through the sweep", async () => { + const storage = new ControlledStorage(); + let opened = await openTasks(storage, [Hold]); + const root = await opened.harness.root(context); + opened.harness.resume(); + const tree = await root.commit(async (tx) => { + const b = await tx.createTask(Hold, { name: "b" }, { ownership: { kind: "conversation" } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: b } }); + return { b, c: c.id }; + }, context); + gate("b").resolve(); + await opened.harness.waitForTask(tree.b, context); + await opened.harness.close(context); + // After reopen, B and C's edge are known only once a chain load reads them. + opened = await openTasks(storage.reopen(), [Hold]); + const harness = opened.harness; + const sizes = () => + (harness as unknown as Record)[ + SCHEDULER_INDEX_SIZES + ]!; + expect(sizes().settled).toBe(0); + const c = (await harness.conversation(tree.c, context))!; + const held = storage.holdCommits(); + const creating = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + await held.entered; + // The admission loaded B; the sweep that load scheduled has run on the line since, and kept it. + await harness.waitForIdle(context); + await flush(); + expect(sizes().settled).toBe(1); + expect(sizes().pins).toBeGreaterThan(0); + held.release(); + await creating; + // Published below a known chain: nothing is left to load, and no pin is left. + expect(sizes().unloaded).toBe(0); + expect(sizes().pins).toBe(0); + expect(await settled(harness.waitForIdle(context))).toBe(false); + harness.resume(); + gate("d").resolve(); + await harness.waitForIdle(context); + await harness.close(context); + }); +}); + +describe("group commit: records a decision copies", () => { + type Probe = { readonly name: string }; + /** Runtimes of running probes, by name, so a test commits for them. */ + const runtimes = new Map>(); + const untilAborted = (signal: AbortSignal) => + new Promise((resolve) => signal.addEventListener("abort", () => resolve(), { once: true })); + /** Holds until aborted, its runtime exposed; fails once its gate opens. Its abort handler holds too. */ + const Probe = defineTask({ + name: "gc.probe", + version: 1, + initial: () => ({ phase: "hold", n: 0 }), + phases: { + hold: async (task, runtime, ctx) => { + runtimes.set(task.input.name, runtime); + const failing = gate(task.input.name).promise.then(() => "fail" as const); + if ((await Promise.race([failing, untilAborted(runtime.signal)])) !== "fail") return; + await runtime.commit( + () => ({ status: "terminal", outcome: { status: "failed", error: { message: "gate" } } }), + ctx, + ); + }, + }, + abort: async (_task, runtime) => untilAborted(runtime.signal), + }); + /** Creates probes `x` and `y` and waits on them `failFast`. */ + const Waiter = defineTask({ + name: "gc.fail-fast", + version: 1, + initial: () => ({ phase: "spawn" }), + phases: { + spawn: async (task, runtime, ctx) => + runtime.commit(async (tx) => { + const on: TaskId[] = []; + for (const name of ["x", "y"]) { + on.push(await tx.createTask(Probe, { name }, { ownership: { kind: "task", taskId: task.id } })); + } + return { status: "waiting", checkpoint: { phase: "done" }, on, policy: "failFast" }; + }, ctx), + done: async (_task, runtime, ctx) => runtime.commit(() => COMPLETED, ctx), + }, + abort: async (_task, runtime) => untilAborted(runtime.signal), + }); + + afterEach(() => { + runtimes.clear(); + }); + + /** Advance probe `name` to `n = 1` while the mark-writing commit is in flight, and run a reconcile pass meanwhile. */ + async function raceMark(storage: ControlledStorage, harness: Harness, name: string, mark: () => Promise) { + const heldMark = storage.holdCommits(); + const marking = mark(); + await heldMark.entered; + let prepared = false; + const advancing = runtimes.get(name)!.commit(() => { + prepared = true; + return { status: "running", checkpoint: { phase: "hold", n: 1 } }; + }, context); + await eventually(() => prepared); + await flush(); + // Global idle schedules a reconcile pass, which reaches the line while both are unpublished. + harness.waitForIdle(context).catch(() => {}); + await flush(); + const heldAdvance = storage.holdCommits(); + heldMark.release(); + await heldAdvance.entered; + await flush(); + heldAdvance.release(); + await Promise.all([marking, advancing]); + await flush(); + } + + it("marks a child in a cascade from its current record, not one its pending commit replaces", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Probe]); + const root = await harness.root(context); + const { parent, child } = await root.commit(async (tx) => { + const parent = await tx.createTask(Probe, { name: "p" }, { ownership: { kind: "conversation" } }); + const child = await tx.createTask(Probe, { name: "c" }, { ownership: { kind: "task", taskId: parent } }); + return { parent, child }; + }, context); + harness.resume(); + await eventually(() => runtimes.has("c")); + await flush(); + await raceMark(storage, harness, "c", () => harness.abortTask(parent, context)); + await eventually(async () => (await harness.getTask(child, context))?.abortRequested === true); + expect((await harness.getTask(child, context))?.state).toMatchObject({ checkpoint: { n: 1 } }); + await harness.close(context); + }); + + it("marks the other tasks of a failFast wait from their current records", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Probe, Waiter]); + const root = await harness.root(context); + await root.commit((tx) => tx.createTask(Waiter, null, { ownership: { kind: "conversation" } }), context); + harness.resume(); + await eventually(() => runtimes.has("x") && runtimes.has("y")); + await flush(); + const y = runtimes.get("y")!.taskId; + // X fails while its commit is in flight; the pass that marks Y runs while Y's own commit is pending. + await raceMark(storage, harness, "y", async () => { + gate("x").resolve(); + }); + await eventually(async () => (await harness.getTask(y, context))?.abortRequested === true); + expect((await harness.getTask(y, context))?.state).toMatchObject({ checkpoint: { n: 1 } }); + await harness.close(context); + }); +}); + +describe("group commit: reads on the line", () => { + it("reads no task record from Storage for more live and awaited tasks than the record cache's count", async () => { + class Counting extends ControlledStorage { + taskReads = 0; + override task(...args: Parameters) { + this.taskReads++; + return super.task(...args); + } + } + const Child = defineTask({ + name: "gc.child", + version: 1, + initial: () => ({ phase: "end" }), + phases: { end: async (task, runtime, ctx) => runtime.commit(() => completedWith(task.id), ctx) }, + abort: async () => {}, + }); + const Parent = defineTask({ + name: "gc.parent", + version: 1, + initial: () => ({ phase: "spawn" }), + phases: { + spawn: async (task, runtime, ctx) => + runtime.commit(async (tx) => { + const on: TaskId[] = []; + for (let n = 0; n < 300; n++) { + on.push(await tx.createTask(Child, null, { ownership: { kind: "task", taskId: task.id } })); + } + return { status: "waiting", checkpoint: { phase: "read", on }, on, policy: "allSettled" }; + }, ctx), + read: async (task, runtime, ctx) => { + const outcomes = await runtime.outcomes(task.state.checkpoint.on, ctx); + const sum = outcomes.reduce((total, outcome) => total + (outcome.status === "completed" ? 1 : 0), 0); + await runtime.commit(() => completedWith(sum), ctx); + }, + }, + abort: async () => {}, + }); + const storage = new Counting(); + const { harness } = await openTasks(storage, [Parent, Child]); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Parent, null, { ownership: { kind: "conversation" } }), + context, + ); + storage.taskReads = 0; + harness.resume(); + expect((await harness.waitForTask(id, context)).state.outcome).toEqual({ status: "completed", result: 300 }); + // Each change of a live task, and the parent's read of the outcomes it waited on, finds the record in memory. + expect(storage.taskReads).toBe(0); + await harness.close(context); + }); +}); + +describe("group commit: outcomes and publication through the Harness", () => { + it("finds the submission an unpublished commit admitted with the same request ID", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const draft = { type: "write", entry: { kind: "note", data: {} }, requestId: "r1" } as const; + const held = storage.holdCommits(); + const first = root.submit(draft, context); + await held.entered; + const second = root.submit(draft, context); + await flush(); + held.release(); + expect((await second).id).toBe((await first).id); + await harness.close(context); + }); + + it("runs a wait continuation of one commit before the next commit of the same call is published", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, [Hold]); + const root = await harness.root(context); + const other = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + const one = await root.commit( + (tx) => tx.createTask(Hold, { name: "one" }, { ownership: { kind: "conversation" } }), + context, + ); + const two = await other.commit( + (tx) => tx.createTask(Hold, { name: "two" }, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await eventually( + async () => (await status(harness, one)) === "running" && (await status(harness, two)) === "running", + ); + const ended = new Set(); + followCommits(harness, (publication) => { + if (publication.kind !== "stored") return; + for (const change of publication.changes) { + if (change.type === "task" && change.value.state.status === "terminal") ended.add(change.value.id); + } + }); + // Both end in one call, behind a commit in flight. + const held = storage.holdCommits(); + const blocker = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + let seen: boolean | undefined; + const waited = harness.waitForTask(one, context).then(() => { + seen = ended.has(two); + }); + gate("one").resolve(); + await flush(); + gate("two").resolve(); + await flush(); + held.release(); + await blocker; + await waited; + await harness.waitForTask(two, context); + expect(seen).toBe(false); + await harness.close(context); + }); + + it("signals an invocation when it closes during a call in flight, and stores that call before Storage closes", async () => { + const storage = new ControlledStorage(); + const signalled = deferred(); + const Listens = defineTask({ + name: "gc.listens", + version: 1, + initial: () => ({ phase: "wait" }), + phases: { + wait: async (_task, runtime) => { + await aborted(runtime.signal).catch(() => signalled.resolve()); + }, + }, + abort: async () => {}, + }); + const { harness } = await openTasks(storage, [Listens]); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Listens, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await eventually(async () => (await status(harness, id)) === "running"); + const held = storage.holdCommits(); + const entry = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + const closing = harness.close(context); + await signalled.promise; + expect(await settled(closing)).toBe(false); + held.release(); + await entry; + await closing; + const reopened = await openTasks(storage.reopen(), [Listens]); + const conversation = (await reopened.harness.conversation(root.id as ConversationId, context))!; + expect((await conversation.entries({}, 10, undefined, context)).items.map((item) => item.kind)).toContain("note"); + // Close wrote the task it stopped back as pending. + expect(await status(reopened.harness, id)).toBe("pending"); + await reopened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-host-inputs.test.ts b/packages/durable/test/harness-host-inputs.test.ts index 4719cea94..4922cd0d4 100644 --- a/packages/durable/test/harness-host-inputs.test.ts +++ b/packages/durable/test/harness-host-inputs.test.ts @@ -6,7 +6,6 @@ import { fauxToolCall, type Models, type SimpleStreamOptions, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -19,6 +18,7 @@ import { MemoryStorage, type Registry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { allEntries, textOf } from "./chat-support.ts"; import { context } from "./session-support.ts"; @@ -86,7 +86,7 @@ describe("HarnessOptions.models and settings as functions", () => { defineTool({ name: "probe", description: "Records api.models before and after the host swaps it", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { seen.push(api.models); current = second; diff --git a/packages/durable/test/harness-idle-scopes.test.ts b/packages/durable/test/harness-idle-scopes.test.ts new file mode 100644 index 000000000..b034150b0 --- /dev/null +++ b/packages/durable/test/harness-idle-scopes.test.ts @@ -0,0 +1,591 @@ +// Idle checks without per-commit scans: a task change settles only the conversations above it (idle waiters and kept +// context ranges), expiry runs oldest first from an ordered index, and `wake()` reads only sleeping tasks while work +// is active. Every case checks the incremental idle state against a full recompute (`SCHEDULER_IDLE_STATE`). +import { createModels } from "@earendil-works/pi-ai"; +import { + type Conversation, + type ConversationId, + createRegistry, + defineExtension, + defineTask, + Harness, + MemoryStorage, + type SetTimer, + type Storage, + type TaskId, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { SCHEDULER_IDLE_STATE } from "../src/harness/harness.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import { ROOT_CONVERSATION_ID, type StorageWrite } from "../src/types.ts"; +import { ControlledStorage, context, flush } from "./session-support.ts"; +import { aborted, type Deferred, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +const gates = new Map>(); +/** Resolves once the named task's context read returned. */ +const reads = new Map>(); + +function named(map: Map>, name: string): Deferred { + let found = map.get(name); + if (found === undefined) { + found = deferred(); + map.set(name, found); + } + return found; +} + +afterEach(() => { + gates.clear(); + reads.clear(); +}); + +/** + * Reads the context of conversation `read` (when given), then holds until its gate opens and completes; its abort + * handler ends it `aborted`. + */ +const Work = defineTask<{ name: string; read?: number }, { phase: "run" }, null>({ + name: "test.idle.work", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + if (task.input.read !== undefined) { + await runtime.context(task.input.read as ConversationId, ctx); + named(reads, task.input.name).resolve(); + } + const opened = await Promise.race([ + named(gates, task.input.name).promise.then(() => true), + aborted(runtime.signal), + ]); + if (opened !== true) return; + await runtime.commit(() => ({ status: "terminal", outcome: { status: "completed", result: null } }), ctx); + }, + }, + abort: async (_task, runtime, ctx) => + runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), ctx), +}); + +type IdleState = Record<"contexts" | "idleContexts" | "idleMismatches" | "fullIdleChecks", number>; + +function idleState(harness: Harness): IdleState { + return (harness as unknown as Record)[SCHEDULER_IDLE_STATE]!; +} + +/** Background timers declined, as Cloudflare Workers do; foreground ones on `setTimeout`. */ +const declineBackground: SetTimer = (callback, delayMs, { background }) => { + if (background) return undefined; + const timer = setTimeout(callback, delayMs); + return () => clearTimeout(timer); +}; + +/** + * A resumed Harness running `Work`, over MemoryStorage counting the entry rows each scan returns, with `clock` as its + * time, the retention `retention` returns, and background timers declined unless `timers`. + */ +async function open(options: { clock?: () => number; retention?: () => number; timers?: boolean } = {}) { + const scanned = { rows: 0 }; + const memory = new MemoryStorage(); + const storage = new Proxy(memory, { + get(target, key) { + const value: unknown = Reflect.get(target, key, target); + if (key === "scanEntries") { + return async (...args: Parameters) => { + const page = await target.scanEntries(...args); + scanned.rows += page.items.length; + return page; + }; + } + return typeof value === "function" ? value.bind(target) : value; + }, + }); + const registry = createRegistry(); + registry.install(defineExtension({ name: "tasks", tasks: [Work] })); + const retention = options.retention ?? (() => 600_000); + const harness = await Harness.open( + storage, + { + models: createModels(), + registry, + settings: () => ({ contextRetentionMs: retention() }), + ...(options.clock === undefined ? {} : { now: options.clock }), + ...(options.timers === true ? {} : { setTimer: declineBackground }), + }, + context, + ); + harness.resume(); + const root = await harness.root(context); + /** A new ownerless conversation with `entries` entries. */ + const conversation = async (entries = 2): Promise => { + const created = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + for (let index = 0; index < entries; index++) { + await created.commit((tx) => tx.appendEntry(created.id, { kind: "note", data: { index } }), context); + } + return created; + }; + /** Start `Work` named `name` in `into`, reading conversation `read`, and wait for its read. */ + const work = async (name: string, into: Conversation, read?: ConversationId): Promise => { + const input = read === undefined ? { name } : { name, read: read as number }; + const id = await into.commit( + (tx) => tx.createTask(Work, input, { ownership: { kind: "conversation" }, conversationId: into.id }), + context, + ); + if (read !== undefined) await named(reads, name).promise; + return id; + }; + /** Open the gate of `name` and wait for its task, `id`, to end. */ + const end = async (name: string, id: TaskId): Promise => { + named(gates, name).resolve(); + await harness.waitForTask(id, context); + }; + /** The entry rows a read of `read` from a new task in `into` scans. */ + const rowsOf = async (name: string, into: Conversation, read: ConversationId): Promise => { + const before = scanned.rows; + const id = await work(name, into, read); + const rows = scanned.rows - before; + await end(name, id); + return rows; + }; + return { harness, root, conversation, work, end, rowsOf }; +} + +/** Flush until scheduler passes started by the last changes settled. */ +async function quiet(): Promise { + for (let round = 0; round < 4; round++) await flush(); +} + +describe("idle waiters by scope", () => { + it("resolve through owner chains, owned conversations and ended tasks between, ignoring background work", async () => { + const { harness, root } = await open(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Work, { name: "p" }, { ownership: { kind: "conversation" } }); + const b = await tx.createTask(Work, { name: "b" }, { ownership: { kind: "task", taskId: p } }); + const a = await tx.createTask(Work, { name: "a" }, { ownership: { kind: "task", taskId: b } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: a } }); + const d = await tx.createTask( + Work, + { name: "d" }, + { ownership: { kind: "conversation" }, conversationId: c.id }, + ); + const g = await tx.createTask( + Work, + { name: "g" }, + { ownership: { kind: "conversation" }, conversationId: c.id, background: true }, + ); + return { p, b, a, c: c.id, d, g }; + }, context); + const c = (await harness.conversation(tree.c, context))!; + const idleC = c.waitForIdle(context); + const idleRoot = root.waitForIdle(context); + const idleAll = harness.waitForIdle(context); + // The owners A and B hold for D: live, though their handlers returned. + named(gates, "a").resolve(); + named(gates, "b").resolve(); + await eventually(async () => (await harness.getTask(tree.b, context))!.state.status === "completing"); + expect([await settled(idleC), await settled(idleRoot), await settled(idleAll)]).toEqual([false, false, false]); + // D ends: C is idle (G is background); A and B finalize; P still holds the root. + named(gates, "d").resolve(); + await idleC; + await harness.waitForTask(tree.b, context); + expect([await settled(idleRoot), await settled(idleAll)]).toEqual([false, false]); + named(gates, "p").resolve(); + await idleRoot; + await idleAll; + expect((await harness.getTask(tree.g, context))!.state.status).not.toBe("terminal"); + await harness.close(context); + }); + + it("count work whose chain is not loaded yet as inside, until the open reconcile loads it", async () => { + const storage = new ControlledStorage(); + const first = await openTasks(storage, [Work]); + const firstRoot = await first.harness.root(context); + const tree = await firstRoot.commit(async (tx) => { + const p = await tx.createTask(Work, { name: "p" }, { ownership: { kind: "conversation" } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: p } }); + const d = await tx.createTask( + Work, + { name: "d" }, + { ownership: { kind: "conversation" }, conversationId: c.id }, + ); + const other = await tx.createConversation({ ownership: { kind: "ownerless" } }); + return { p, c: c.id, d, other: other.id }; + }, context); + await first.harness.close(context); + // Reopened, no chain is loaded: P and D count as inside every scope until the reconcile pass loads theirs. + const { harness } = await openTasks(storage.reopen(), [Work]); + const idleOther = (await harness.conversation(tree.other, context))!.waitForIdle(context); + const idleRoot = (await harness.root(context)).waitForIdle(context); + await idleOther; + expect(await settled(idleRoot)).toBe(false); + harness.resume(); + named(gates, "d").resolve(); + await harness.waitForTask(tree.d, context); + expect(await settled(idleRoot)).toBe(false); + named(gates, "p").resolve(); + await idleRoot; + await harness.close(context); + }); + + it("resolve a waiter on an unrelated conversation without settling the busy one", async () => { + const { harness, conversation, work, end } = await open(); + const [busy, other] = [await conversation(0), await conversation(0)]; + const held = await work("held", busy); + const quick = await work("quick", other); + const idleBusy = busy.waitForIdle(context); + const idleOther = other.waitForIdle(context); + await end("quick", quick); + await idleOther; + expect(await settled(idleBusy)).toBe(false); + await end("held", held); + await idleBusy; + await harness.close(context); + }); +}); + +describe("kept context ranges", () => { + it("settles a run end whose sweep drops the chain without a check of every scope", async () => { + const { harness, root } = await open(); + const tree = await root.commit(async (tx) => { + const p = await tx.createTask(Work, { name: "p" }, { ownership: { kind: "conversation" } }); + const t = await tx.createTask(Work, { name: "t" }, { ownership: { kind: "task", taskId: p } }); + const c = await tx.createConversation({ ownership: { kind: "task", taskId: t } }); + // A context range keeps entries: an empty conversation has none to keep. + await tx.appendEntry(c.id, { kind: "note" }); + return { p, t, c: c.id }; + }, context); + const c = (await harness.conversation(tree.c, context))!; + const d = await c.commit( + (tx) => + tx.createTask( + Work, + { name: "d", read: tree.c as number }, + { ownership: { kind: "conversation" }, conversationId: tree.c }, + ), + context, + ); + await named(reads, "d").promise; + named(gates, "t").resolve(); + await eventually(async () => (await harness.getTask(tree.t, context))!.state.status === "completing"); + await quiet(); + const before = idleState(harness); + expect(before).toMatchObject({ contexts: 1, idleContexts: 0, idleMismatches: 0 }); + // D's end makes C idle, T finalizes, and the sweep forgets C's chain: none of it checks every scope. + named(gates, "d").resolve(); + await harness.waitForTask(d, context); + await harness.waitForTask(tree.t, context); + await quiet(); + expect(idleState(harness)).toEqual({ ...before, idleContexts: 1 }); + named(gates, "p").resolve(); + await harness.waitForTask(tree.p, context); + await harness.close(context); + }); + + it("drops a range past its retention when a task change makes it busy again, with background timers declined", async () => { + let clock = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => 1_000 }); + const [x, z] = [await conversation(), await conversation(0)]; + await end("reader", await work("reader", z, x.id)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1, idleMismatches: 0 }); + clock = 2_500; + // No timer expired it; the task change that makes X busy checks expiry first. + const busy = await work("busy", x); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleContexts: 0, idleMismatches: 0 }); + await end("busy", busy); + await harness.close(context); + }); + + it("drops a range past its retention at a reconcile pass with nothing to do, with background timers declined (review)", async () => { + let clock = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => 1_000 }); + const [x, z] = [await conversation(), await conversation(0)]; + await end("reader", await work("reader", z, x.id)); + const held = await work("held", z); + // Held runs: no task changes from here. + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1 }); + clock = 2_500; + // A wait on busy Z asks for a reconcile pass, which has no work and changes no task: it checks expiry. + const idle = z.waitForIdle(context); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleMismatches: 0 }); + await end("held", held); + await idle; + await harness.close(context); + }); + + it("expires in idle order when the clock goes back", async () => { + let clock = 1_000; + const { harness, conversation, work, end, rowsOf } = await open({ + clock: () => clock, + retention: () => 1_000, + }); + const [a, b, z] = [await conversation(3), await conversation(5), await conversation(0)]; + await end("read a", await work("read a", z, a.id)); + clock = 500; + await end("read b", await work("read b", z, b.id)); + expect(idleState(harness)).toMatchObject({ contexts: 2, idleContexts: 2, idleMismatches: 0 }); + // B went idle at 500 and expires first, though it went idle last. + clock = 1_600; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleMismatches: 0 }); + expect(await rowsOf("again b", z, b.id)).toBe(5); + expect(await rowsOf("again a", z, a.id)).toBe(0); + await harness.close(context); + }); + + it("drops every idle range when retention becomes zero, also one that went idle after the clock's now (review)", async () => { + let clock = 2_000; + let retention = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => retention }); + const [a, z] = [await conversation(), await conversation(0)]; + await end("read", await work("read", z, a.id)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1 }); + // The clock goes back below A's idle start; then retention 0 and an unrelated task change. + clock = 1_000; + retention = 0; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); + + it("keeps an idle range's expiry when another task reads it again", async () => { + let clock = 1_000; + const { harness, conversation, work, end, rowsOf } = await open({ + clock: () => clock, + retention: () => 1_000, + }); + const [a, b, z] = [await conversation(3), await conversation(5), await conversation(0)]; + await end("read a", await work("read a", z, a.id)); + clock = 1_500; + await end("read b", await work("read b", z, b.id)); + clock = 1_800; + expect(await rowsOf("again a", z, a.id)).toBe(0); + clock = 2_100; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleMismatches: 0 }); + expect(await rowsOf("late b", z, b.id)).toBe(0); + await harness.close(context); + }); + + it("drops a range from the expiry order when an entry commit finds it expired", async () => { + let clock = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => 1_000 }); + const [a, z] = [await conversation(), await conversation(0)]; + await end("read", await work("read", z, a.id)); + clock = 2_500; + // An entry-only commit settles no idleness: the range is dropped where it would be extended. + await a.commit((tx) => tx.appendEntry(a.id, { kind: "note", data: { late: true } }), context); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleContexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); + + it("drops a range from the expiry order when an entry commit cannot extend it", async () => { + const { harness, conversation, work, end } = await open(); + const [a, z] = [await conversation(0), await conversation(0)]; + const first = await a.commit((tx) => tx.appendEntry(a.id, { kind: "note" }), context); + const second = await a.commit((tx) => tx.appendEntry(a.id, { kind: "note" }), context); + await a.commit((tx) => tx.appendEntry(a.id, { kind: "summary", head: second.id }), context); + await end("read", await work("read", z, a.id)); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 1, idleMismatches: 0 }); + // A marker whose head is older than the range's: the range cannot be extended and is dropped. + await a.commit((tx) => tx.appendEntry(a.id, { kind: "summary", head: first.id }), context); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleContexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); + + it.each([ + ["shorter", 400, 0], + ["longer", 10_000, 1], + ["zero", 0, 0], + ] as const)("applies a %s retention to idle ranges at an unrelated commit", async (_name, changed, kept) => { + let clock = 1_000; + let retention = 1_000; + const { harness, conversation, work, end } = await open({ clock: () => clock, retention: () => retention }); + const [a, z] = [await conversation(), await conversation(0)]; + await end("read", await work("read", z, a.id)); + clock = 1_500; + retention = changed; + await end("unrelated", await work("unrelated", z)); + expect(idleState(harness)).toMatchObject({ contexts: kept, idleMismatches: 0 }); + await harness.close(context); + }); + + it("drops a range that goes idle with zero retention", async () => { + const { harness, conversation, work, end } = await open({ retention: () => 0 }); + const a = await conversation(); + const own = await work("own", a, a.id); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 1, idleContexts: 0, idleMismatches: 0 }); + await end("own", own); + await quiet(); + expect(idleState(harness)).toMatchObject({ contexts: 0, idleMismatches: 0 }); + await harness.close(context); + }); +}); + +/** A random source with a fixed seed. */ +function seeded(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1103515245 + 12345) % 2147483648; + return state / 2147483648; + }; +} + +describe("incremental idle state", () => { + it("equals a full recompute after every step of random nested work", async () => { + const random = seeded(42); + const pick = (items: readonly T[]): T => items[Math.floor(random() * items.length)]!; + const { harness, root } = await open(); + // A context range keeps entries: every conversation gets one. + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + const conversations: ConversationId[] = [root.id]; + const live = new Map(); + let count = 0; + let fullChecks = idleState(harness).fullIdleChecks; + const handle = async (id: ConversationId) => (await harness.conversation(id, context))!; + for (let step = 0; step < 200; step++) { + const action = random(); + const name = `t${count++}`; + if (action < 0.1) { + const created = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); + await created.commit((tx) => tx.appendEntry(created.id, { kind: "note" }), context); + conversations.push(created.id); + } else if (action < 0.45 || live.size === 0) { + // A task in any conversation, also one whose owner chain ended and was swept; it may read a context. + const into = pick(conversations); + const read = random() < 0.6 ? pick(conversations) : undefined; + const background = random() < 0.15; + const input = read === undefined ? { name } : { name, read: read as number }; + const target = await handle(into); + const id = await target.commit( + (tx) => + tx.createTask(Work, input, { ownership: { kind: "conversation" }, conversationId: into, background }), + context, + ); + live.set(name, { id, conversation: into }); + if (read !== undefined) await named(reads, name).promise; + } else if (action < 0.6) { + // Work a live task owns: a child task, or an owned conversation with a task that reads it. + const [ownerName, owner] = pick([...live]); + const state = (await harness.getTask(owner.id, context))?.state.status; + if (state === undefined || state === "terminal" || state === "completing") { + live.delete(ownerName); + continue; + } + const conversation = random() < 0.5; + const created = await (await handle(owner.conversation)).commit(async (tx) => { + if (!conversation) { + const id = await tx.createTask(Work, { name }, { ownership: { kind: "task", taskId: owner.id } }); + return { id, into: owner.conversation, read: false }; + } + const child = await tx.createConversation({ ownership: { kind: "task", taskId: owner.id } }); + await tx.appendEntry(child.id, { kind: "note" }); + const id = await tx.createTask( + Work, + { name, read: child.id as number }, + { ownership: { kind: "conversation" }, conversationId: child.id }, + ); + return { id, into: child.id, read: true }; + }, context); + if (created.read) conversations.push(created.into); + live.set(name, { id: created.id, conversation: created.into }); + if (created.read) await named(reads, name).promise; + } else { + const [ended] = pick([...live]); + live.delete(ended); + named(gates, ended).resolve(); + } + await quiet(); + const state = idleState(harness); + expect(state.idleMismatches, `step ${step}`).toBe(0); + fullChecks = state.fullIdleChecks; + } + const state = idleState(harness); + expect(state.contexts).toBeGreaterThan(5); + expect(state.idleContexts).toBeGreaterThan(0); + expect(state.idleContexts).toBeLessThan(state.contexts); + expect(fullChecks).toBe(0); + for (const gate of gates.values()) gate.resolve(); + await harness.close(context); + }); +}); + +describe("wake()", () => { + const kind = Work.definition.name; + const record = (id: number, state: unknown, extra: Record = {}): StorageWrite => ({ + type: "task", + value: { + id, + conversationId: ROOT_CONVERSATION_ID, + kind, + version: 1, + input: { name: `seeded${id}` }, + state, + background: false, + abortRequested: false, + ...extra, + } as never, + }); + const sleeping = (until: number) => ({ status: "sleeping", checkpoint: { phase: "run" }, until }); + const SLEEPS = record(10, sleeping(5_000)); + + it.each([ + ["a sleeping task", [SLEEPS], { active: false, at: 5_000 }], + [ + "an abort-marked sleeping task, which runs its handler", + [SLEEPS, record(11, sleeping(3_000), { abortRequested: true })], + { active: true, at: 5_000 }, + ], + ["a sleeping task that is due", [record(11, sleeping(500))], { active: true }], + [ + "a task waiting on a live one", + [SLEEPS, record(11, { status: "waiting", checkpoint: { phase: "run" }, on: [10], policy: "allSettled" })], + { active: false, at: 5_000 }, + ], + [ + "a completing task holding for a sleeping child", + [ + record(11, { status: "completing", outcome: { status: "completed", result: null } }), + record(10, sleeping(5_000), { owner: 11 }), + ], + { active: false, at: 5_000 }, + ], + [ + "a task no definition can take", + [SLEEPS, record(11, { status: "pending", checkpoint: { phase: "run" } }, { kind: "test.idle.missing" })], + { active: false, at: 5_000 }, + ], + [ + "a pending task", + [SLEEPS, record(11, { status: "pending", checkpoint: { phase: "run" } })], + { active: true, at: 5_000 }, + ], + ] as const)("with %s", async (_name, writes, expected) => { + const storage = new MemoryStorage(); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, ...writes], context); + const { harness } = await openTasks(storage, [Work], { now: () => 1_000 }); + await quiet(); + expect(harness.wake()).toEqual(expected); + await harness.close(context); + }); + + it("with an invocation running: active, and the earliest sleep still found", async () => { + const storage = new MemoryStorage(); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, SLEEPS], context); + const { harness } = await openTasks(storage, [Work], { now: () => 1_000 }); + harness.resume(); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Work, { name: "running" }, { ownership: { kind: "conversation" } }), + context, + ); + await eventually(async () => (await harness.getTask(id, context))!.state.status === "running"); + expect(harness.wake()).toEqual({ active: true, at: 5_000 }); + named(gates, "running").resolve(); + await harness.waitForTask(id, context); + await quiet(); + expect(harness.wake()).toEqual({ active: false, at: 5_000 }); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-inbox.test.ts b/packages/durable/test/harness-inbox.test.ts index a96ab0f6c..a4cb91837 100644 --- a/packages/durable/test/harness-inbox.test.ts +++ b/packages/durable/test/harness-inbox.test.ts @@ -9,12 +9,10 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, type Usage, } from "@earendil-works/pi-ai"; import { type Conversation, - defineDoc, defineTool, type EntryRecord, GenerationTask, @@ -30,6 +28,7 @@ import { ToolTask, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { recordUsage } from "../src/harness/usage.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; @@ -68,7 +67,7 @@ function holdTool(setup: ChatSetup, gate: Deferred, result: ToolExecutionResult defineTool({ name: "hold", description: "Waits for the test", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return result; @@ -191,22 +190,20 @@ describe("inbox", () => { await first.reached; const f1 = await root.submit({ type: "input", content: "f1" }, context); const f2 = await root.submit({ type: "input", content: "f2" }, context); - // Occupy the line, let the answer queue its boundary commit behind it, then change the mode. - const held = storage.holdCommits(); - const Marker = defineDoc<{ n: number }>({ - checkpointWhen: () => false, - kind: "test.marker", - version: 1, - scope: "session", - initial: () => ({ n: 0 }), - }); - const occupying = root.commit(async (tx) => void (await tx.doc(Marker)).n++, context); - await held.entered; + // Occupy the line with a commit whose callback waits, let the answer queue its boundary commit behind it, then + // change the mode. + const entered = deferred(); + const release = deferred(); + const occupying = root.commit(async () => { + entered.resolve(); + await release.promise; + }, context); + await entered.promise; first.release(); await yielded.promise; await flush(); setup.settings.followUpMode = "all"; - held.release(); + release.resolve(); await occupying; const settled = await f2.wait(context); expect(await status(f1)).toEqual({ ...settled, id: f1.id, entry: expect.any(Number) }); @@ -477,7 +474,7 @@ describe("inbox", () => { defineTool({ name: "hold", description: "Hands off", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [], control: { handoff: `h${++handoffs}` } }), }), ); @@ -799,7 +796,7 @@ describe("inbox", () => { defineTool({ name: "later", description: "Finishes first", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await secondGate.promise; return { output: [], control: { handoff: "two" } }; @@ -898,17 +895,17 @@ describe("inbox", () => { let inboxId: number | undefined; class RecordingStorage extends MemoryStorage { override async commit( - writes: Parameters[0], - adopt: Parameters[1], + batches: Parameters[0], + receive: Parameters[1], callContext: Parameters[2], ) { - for (const write of writes) { + for (const write of batches.flat()) { if (write.type !== "document.change" || write.id !== inboxId) continue; const content = write.content; const empty = content.kind === "base" && (content.value as InboxState).items.length === 0; written.push({ kind: content.kind, empty }); } - return super.commit(writes, adopt, callContext); + return super.commit(batches, receive, callContext); } } const setup = chatSetup(); diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index f0c44b9fb..ce6baa468 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -1,4 +1,5 @@ // Inputs placed as application entries, and data-only writes appended at once (#10407). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,7 +10,6 @@ import { fauxText, fauxToolCall, type Context as ModelContext, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -23,6 +23,7 @@ import { MemoryStorage, type SubmissionDraft, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; @@ -54,7 +55,7 @@ function holdTool(setup: ChatSetup, gate: Deferred): void { defineTool({ name: "hold", description: "Waits for the test", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return { output: [{ type: "text", text: "held" }] }; diff --git a/packages/durable/test/harness-lifecycle.test.ts b/packages/durable/test/harness-lifecycle.test.ts index 0c0049599..bf044120b 100644 --- a/packages/durable/test/harness-lifecycle.test.ts +++ b/packages/durable/test/harness-lifecycle.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context, JsonValue } from "@earendil-works/chord"; import { withAbortSignal } from "@earendil-works/chord/context"; -import { createModels, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { createModels, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationId, @@ -17,9 +17,11 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; import { ControlledStorage, context, flush } from "./session-support.ts"; @@ -73,12 +75,12 @@ async function seedRunningTask(storage: Storage): Promise { const id = await storage.mintId>(); const state = { status: "running", checkpoint: { phase: "run" } } as const; const task = { id, conversationId, kind: "test.seeded", version: 1, input: null, background: false, state }; - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: conversationId } }, { type: "task", value: { ...task, abortRequested: false } }, ], - () => {}, context, ); } @@ -167,7 +169,7 @@ describe("Harness close", () => { defineTool({ name: "wait", description: "wait", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { if (where === "tool") await stubborn(); return { output: [] }; diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 518399392..93ac8187c 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -1,5 +1,5 @@ import type { Op } from "@earendil-works/chord/delta"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type CommitPublication, defineTool, @@ -10,6 +10,7 @@ import { type ToolExecutionApi, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -30,7 +31,7 @@ async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { defineTool({ name: "drive", description: "Driven by the test", - parameters: Type.Object({}), + parameters: obj({}), outputLimits, execute: async (_args, api, callContext) => { for (;;) { @@ -104,7 +105,7 @@ describe("pi.live deltas", () => { defineTool({ name: "noop", description: "noop", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -138,36 +139,32 @@ describe("pi.live deltas", () => { { taskId: firstGeneration, inputs: [expect.any(Number)], turns: 0, first: expect.any(Number) }, ], ], - // request - [["s", ["generation"], { attempt: 1 }]], - // the generation starts its tool round and keeps the run, counting the round as one turn (#11043) - expect.arrayContaining([ - ["d", ["generation"]], + // the generation starts its tool round and keeps the run, counting the round as one turn (#11043); the request's + // attempt is deferred, so it is stored here, its held operation first + [ + ["s", ["generation"], { attempt: 1 }], ["s", ["tools"], [{ callId: "c1", name: "noop", taskId: tool, status: "pending" }]], + ["d", ["generation"]], ["s", ["run", "turns"], 1], - ]), - // intent - [["s", ["tools", 0, "status"], "running"]], - // result - expect.arrayContaining([ + ], + // result: the running status is deferred like progress, so it is stored with the result, folded into its write + [ ["s", ["tools", 0, "status"], "done"], ["s", ["tools", 0, "entry"], result], - ]), + ], // the generation's tools phase hands the run to the next generation expect.arrayContaining([ ["d", ["tools"]], ["s", ["run", "taskId"], secondGeneration], ]), - [["s", ["generation"], { attempt: 1 }]], - // the answer ends the run - expect.arrayContaining([ + // the answer ends the run, with the second request's deferred attempt first + [ + ["s", ["generation"], { attempt: 1 }], ["d", ["run"]], ["d", ["generation"]], - ]), + ], ]); - expect(commits[2]).toHaveLength(3); - expect(commits[4]).toHaveLength(2); - expect(commits[5]).toHaveLength(2); + expect(commits[3]).toHaveLength(2); await harness.close(context); }); @@ -292,17 +289,26 @@ describe("pi.live deltas", () => { let liveId: number | undefined; class RecordingStorage extends MemoryStorage { override async commit( - writes: Parameters[0], - adopt: Parameters[1], + batches: Parameters[0], + receive: Parameters[1], context: Parameters[2], ) { - let kind: "base" | "delta" | undefined; - for (const write of writes) { - if (write.type === "document.change" && write.id === liveId) kind = write.content.kind; - } - const seq = await super.commit(writes, adopt, context); - if (kind !== undefined) written.set(seq, kind); - return seq; + const kinds = batches.map((writes) => { + let kind: "base" | "delta" | undefined; + for (const write of writes) { + if (write.type === "document.change" && write.id === liveId) kind = write.content.kind; + } + return kind; + }); + return super.commit( + batches, + (index, seq) => { + const kind = kinds[index]; + if (kind !== undefined) written.set(seq, kind); + receive(index, seq); + }, + context, + ); } } const setup = chatSetup(); @@ -312,7 +318,7 @@ describe("pi.live deltas", () => { defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { api.output(`${name} output\n`); await new Promise((resolve) => setTimeout(resolve, 150)); @@ -329,6 +335,8 @@ describe("pi.live deltas", () => { fauxAssistantMessage([fauxText("done")]), ]); setup.settings.toolExecution = "sequential"; + // Store deferred progress within each tool's run, so running slots reach Storage as deltas. + setup.settings.progress = { persistWithinMs: 50 }; const { harness, root } = await openChat(new RecordingStorage(), setup); const values = new Map(); harness.observeCommits({ @@ -361,7 +369,7 @@ describe("pi.live deltas", () => { defineTool({ name: "bad", description: "bad", - parameters: Type.Object({}), + parameters: obj({}), // Not strict JSON: the result commit throws and the scheduler faults the task. execute: async () => ({ output: [], details: { fn: (() => 1) as never } }), }), @@ -405,7 +413,7 @@ describe("pi.live deltas", () => { defineTool({ name: "noop", description: "noop", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 0eb29ff76..8d80aa903 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineExtension, @@ -23,6 +22,7 @@ import { ToolTask, type ToolTaskInput, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; @@ -31,7 +31,7 @@ import { nestedResultOf, nestedResults, viewHistory } from "./live-support.ts"; import { context } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); function tool( name: string, diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index e1442b25f..66de32766 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -10,7 +10,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, type Usage, } from "@earendil-works/pi-ai"; import { @@ -33,15 +32,17 @@ import { type ToolTaskResult, UsageDoc, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; import { endedAt, nestedResultOf, nestedResults, slotsOf, startOrder, viewHistory } from "./live-support.ts"; import { context, onValue } from "./session-support.ts"; import { aborted, deferred, eventually } from "./task-support.ts"; -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); type Tool = ToolRegistration; function tool(name: string, execute: Tool["execute"], extra: Partial = {}): Tool { @@ -117,11 +118,11 @@ describe("nested tool calls", () => { // Per document written, the keys of the nested calls it holds. const written: { kind: string; key: string | undefined }[] = []; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - for (const write of writes) { + storage.commit = (batches, receive, commitContext) => { + for (const write of batches.flat()) { if (write.type === "document.create") written.push({ kind: write.record.kind, key: write.record.key }); } - return commit(writes, adopt, commitContext); + return commit(batches, receive, commitContext); }; const { harness, root } = await openChat(storage, setup); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -328,7 +329,7 @@ describe("nested tool calls", () => { defineTool({ name: "pair", description: "pair", - parameters: Type.Object({ a: Type.String(), b: Type.String() }), + parameters: obj({ a: str(), b: str() }), execute: async (args) => { runs++; return { output: [{ type: "text", text: `${args.a}${args.b}` }] }; @@ -1280,9 +1281,9 @@ describe("tool task version 1 and 2 records", () => { const { kind: _kind, ...version1 } = record.input as Extract; const input = version === 1 ? version1 : record.input; const state = { ...record.state, checkpoint } as typeof record.state; - await storage.commit( + await commitWrites( + storage, [{ type: "task", value: { ...record, version, input, state } as typeof record }], - () => {}, context, ); await storage.close(context); diff --git a/packages/durable/test/harness-observers.test.ts b/packages/durable/test/harness-observers.test.ts index fdd6d7787..13b8cbf14 100644 --- a/packages/durable/test/harness-observers.test.ts +++ b/packages/durable/test/harness-observers.test.ts @@ -95,7 +95,7 @@ const collect = (() => { })(); describe("observeConversation and observeTaskGraph", () => { - it("start at the current value and advance inside adopt with the same change a watch gets", async () => { + it("start at the current value and advance at the publication with the same change a watch gets", async () => { const storage = new ControlledStorage(); const { harness } = await openTasks(storage, []); const root = await harness.root(context); @@ -113,7 +113,8 @@ describe("observeConversation and observeTaskGraph", () => { const entry = await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); expect(view.calls).toHaveLength(1); const [call] = view.calls as [{ advance: WatchValue; adopting: boolean }]; - expect(call.adopting).toBe(true); + // Published once Storage stored it, outside its call. + expect(call.adopting).toBe(false); expect(call.advance.retired).toBe(false); expect(call.advance.value.entries.at(-1)?.id).toBe(entry.id); await flush(); @@ -124,8 +125,11 @@ describe("observeConversation and observeTaskGraph", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); expect(view.calls).toHaveLength(1); await watch.stop(); + const graphWatch = await harness.watchTaskGraph(context); await harness.close(context); - expect(graph.calls).toEqual([{ end: { kind: "ended" } }]); + // S1: a Session close fails an observer (it may follow the next Session); a watch ends `ended`. + expect(graph.calls).toEqual([{ end: { kind: "failed", error: new Error("session closed") } }]); + expect(await graphWatch.closed).toEqual({ kind: "ended" }); }); it("end with the error that failed the Harness", async () => { @@ -235,7 +239,7 @@ describe("observeConversation and observeTaskGraph", () => { ops: [["s", ["tasks", key], expect.objectContaining({ blocked: "missing_task" })]], retired: false, }, - adopting: true, + adopting: false, }, ]); registry.install(defineExtension({ name: "late", tasks: [Missing] })); @@ -301,7 +305,7 @@ describe("task documents", () => { await harness.close(context); }); - it("an observer gets the retirement, then its end, inside the adopt of the commit that ends the task", async () => { + it("an observer gets the retirement, then its end, at the publication of the commit that ends the task", async () => { const { harness, storage, id, finish } = await openWriter(); const progress = recorder>(storage); let endedAdopting: boolean | undefined; @@ -316,10 +320,10 @@ describe("task documents", () => { finish(); await harness.waitForTask(id, context); expect(progress.calls).toEqual([ - { advance: { kind: "value", value: { lines: ["started"] }, ops: [], retired: true }, adopting: true }, + { advance: { kind: "value", value: { lines: ["started"] }, ops: [], retired: true }, adopting: false }, { end: { kind: "ended" } }, ]); - expect(endedAdopting).toBe(true); + expect(endedAdopting).toBe(false); await harness.close(context); expect(progress.calls).toHaveLength(2); }); diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index 86bd715b2..d0251e24c 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -1,7 +1,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationHandle, @@ -16,14 +16,16 @@ import { LiveDoc, MemoryStorage, type Storage, + type StorageWrite, type Submission, type TaskId, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, refusingAt } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; /** Outcome a held task commits once its gate opens. */ @@ -937,18 +939,25 @@ describe("ownership", () => { await opened.harness.abortTask(tree.owner, context); await opened.harness.close(context); - const storage = await openNodeSqliteStorage(path); + const storage: Storage = await openNodeSqliteStorage(path); let rejectMark: TaskId | undefined; const commit = storage.commit.bind(storage); - storage.commit = async (writes, adopt, ctx) => { - if ( - writes.some((write) => write.type === "task" && write.value.id === rejectMark && write.value.abortRequested) - ) { - rejectMark = undefined; - throw new Error("disk gone"); - } - return commit(writes, adopt, ctx); - }; + // The batch that marks `rejectMark` is refused. + storage.commit = (batches, receive, ctx) => + refusingAt( + commit, + batches, + receive, + ctx, + (writes) => { + const marks = writes.some( + (write) => write.type === "task" && write.value.id === rejectMark && write.value.abortRequested, + ); + if (marks) rejectMark = undefined; + return marks; + }, + new Error("disk gone"), + ); opened = await openHarness(storage); const child = (await opened.harness.conversation(tree.child, context))!; const late = await child.commit(async (tx) => { @@ -971,19 +980,19 @@ describe("ownership", () => { const failure = new Error("disk gone"); let failMark: TaskId | undefined; class Failing extends ControlledStorage { - override async commit( - writes: Parameters[0], - adopt: Parameters[1], + override commit( + batches: Parameters[0], + receive: Parameters[1], ctx: Parameters[2], ) { - const marks = writes.some( - (write) => write.type === "task" && write.value.id === failMark && write.value.abortRequested, - ); - if (marks) { - failMark = undefined; - throw failure; - } - return super.commit(writes, adopt, ctx); + const refuses = (writes: readonly StorageWrite[]): boolean => { + const marks = writes.some( + (write) => write.type === "task" && write.value.id === failMark && write.value.abortRequested, + ); + if (marks) failMark = undefined; + return marks; + }; + return refusingAt((...args) => super.commit(...args), batches, receive, ctx, refuses, failure); } } const storage = new Failing(); @@ -1011,7 +1020,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1056,7 +1065,7 @@ describe("owned conversations from tools and supervisors", () => { await harness.close(context); }); - it("rejects a handle operation queued on the line when the invocation ends before it runs", async () => { + it("runs a handle operation queued behind the abort mark before the mark's publication, and the cascade aborts it", async () => { const setup = chatSetup(); const ready = deferred<{ child: ConversationId; taskId: TaskId }>(); const go = deferred(); @@ -1066,7 +1075,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates late", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1087,9 +1096,29 @@ describe("owned conversations from tools and supervisors", () => { setup.faux.setResponses([ fauxAssistantMessage([fauxToolCall("delegate", {}, { id: "c1" })], { stopReason: "toolUse" }), ]); - const { harness, root } = await openChat(new MemoryStorage(), setup); + // Holds the storage call that writes the abort mark, so the mark is published only once the test lets it. + let holdMark: TaskId | undefined; + const stored = deferred(); + class HoldMark extends MemoryStorage { + override async commit( + batches: Parameters[0], + receive: Parameters[1], + ctx: Parameters[2], + ) { + const marks = batches + .flat() + .some((write) => write.type === "task" && write.value.id === holdMark && write.value.abortRequested); + if (marks) { + holdMark = undefined; + await stored.promise; + } + return super.commit(batches, receive, ctx); + } + } + const { harness, root } = await openChat(new HoldMark(), setup); await root.submit({ type: "input", content: "go" }, context); const { child, taskId } = await ready.promise; + holdMark = taskId; // Hold the Session line, queue the abort mark behind it, then let the tool queue its submit. const release = deferred(); const holding = root.commit(async () => { @@ -1100,10 +1129,22 @@ describe("owned conversations from tools and supervisors", () => { const { submitting } = await queued.promise; release.resolve(); await holding; + // The submit starts while the mark's storage call is in flight. + await new Promise((resolve) => setTimeout(resolve, 0)); + stored.resolve(); await aborting; - await expect(submitting).rejects.toThrow(); + // Cancellation is checked when a job starts: the submit touches nothing the mark writes, so it starts before the + // mark is published and lands. The mark's cascade then aborts the run it started in the owned conversation. + const submission = (await submitting) as Submission; const childConversation = (await harness.conversation(child, context))!; - expect((await childConversation.entries({}, 10, undefined, context)).items).toEqual([]); + await childConversation.waitForIdle(context); + expect(await (await harness.submission(submission.id, context))!.status(context)).toMatchObject({ + status: "unanswered", + reason: "aborted", + }); + expect((await childConversation.entries({}, 10, undefined, context)).items.map((entry) => entry.kind)).toEqual([ + "pi.user", + ]); expect((await harness.inspect(context)).submissions.filter((record) => record.conversationId === child)).toEqual( [], ); @@ -1119,7 +1160,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1168,7 +1209,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "delegate", description: "Delegates and cancels", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1207,7 +1248,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "spawn", description: "Starts owned work, then throws or hangs", - parameters: Type.Object({ mode: Type.String() }), + parameters: obj({ mode: str() }), execute: async (args, api, callContext) => { const child = await api.commit(async (tx) => { const created = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1265,7 +1306,7 @@ describe("owned conversations from tools and supervisors", () => { defineTool({ name: "subagent", description: "Delegates", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, callContext) => { const child = await api.commit(async (tx) => { diff --git a/packages/durable/test/harness-prompt.test.ts b/packages/durable/test/harness-prompt.test.ts index 40ef2e09e..39affbc58 100644 --- a/packages/durable/test/harness-prompt.test.ts +++ b/packages/durable/test/harness-prompt.test.ts @@ -1,4 +1,4 @@ -import { type SystemMessage, type Tool, Type } from "@earendil-works/pi-ai"; +import type { SystemMessage, Tool } from "@earendil-works/pi-ai"; import { getCurrentTools, toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript"; import { type Conversation, @@ -12,6 +12,7 @@ import { type ToolRegistration, wrapSection, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { resolveAgent, resolveSettings } from "../src/harness/agent.ts"; import { planSystemEntries, renderSections, replaySections } from "../src/harness/prompt.ts"; @@ -207,7 +208,7 @@ describe("tool loadout preparation", () => { const declaration = (name: string, description = name): Tool => ({ name, description, - parameters: Type.Object({}), + parameters: obj({}), }); /** Plan tools only, append the plan, check that replay offers `tools` in order, and return each message's changes. */ diff --git a/packages/durable/test/harness-registry-reader.test.ts b/packages/durable/test/harness-registry-reader.test.ts new file mode 100644 index 000000000..b2a82da35 --- /dev/null +++ b/packages/durable/test/harness-registry-reader.test.ts @@ -0,0 +1,198 @@ +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { + arr, + createHost, + defineFacet, + defineRegistry, + type FacetHost, + local, + method, + type Native, + type RegistryTable, + str, +} from "@earendil-works/chord/next"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { describe, expect, it } from "vitest"; +import { + createRegistry, + defineExtension, + defineTask, + type Extension, + Harness, + MemoryStorage, + type RegistrySnapshot, + ROOT_CONVERSATION_ID, + registryReader, +} from "../src/index.ts"; + +/** p3's pattern: the extensions registry composes Durable's snapshot, leaving out an extension Durable refuses. */ +const Extensions = defineRegistry("app.harness.extensions", { version: 1 }); + +function compose(table: RegistryTable, report: (conflict: { message: string; plugins: string[] }) => void) { + const registry = createRegistry(); + for (const { value, plugin } of table) { + try { + registry.install(value); + } catch (error) { + report({ message: (error as Error).message, plugins: [plugin] }); + } + } + return registry.snapshot(); +} + +const Done = defineTask({ + name: "app.done", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (_task, runtime, taskContext) => { + await runtime.commit( + () => ({ status: "terminal", outcome: { status: "completed", result: null } }), + taskContext, + ); + }, + }, + abort: async () => {}, +}); + +/** A host-lifetime object contributing an extension it replaces with `install` (one with the task `app.done`). */ +const Plugin = local("app.plugin", { version: 1, params: {}, members: { install: method() } }); + +function serve(host: FacetHost): void { + host.serve(Extensions, { compose: (table, { report }) => compose(table, report) }); + host.serve(Plugin, (_, self) => { + const extension = self.writable(defineExtension({ name: "app.plugin" })); + self.contribute(Extensions, extension); + return { + install: async () => { + extension.set(defineExtension({ name: "app.plugin", tasks: [Done] })); + return null; + }, + }; + }); +} + +describe("registryReader", () => { + it("reads the composed snapshot; a listener is called synchronously, inside the batch that composed", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serve) }], context); + const value = await host.registry(Extensions, context); + const reader = registryReader(value, context); + expect(reader.snapshot().extension("app.plugin")?.tasks).toBeUndefined(); + const calls: string[] = []; + const unsubscribe = reader.subscribe(() => { + // Inside Chord's batch: a Chord call here is refused, so a listener only wakes its scheduler. + try { + value.observe({ advance: () => {}, end: () => {} }, context); + calls.push("outside"); + } catch (error) { + calls.push((error as { reason?: string }).reason ?? "other"); + } + }); + const plugin = await host.get(Plugin, {}, context); + await plugin.install(context); + expect(calls).toEqual(["reentrant"]); + expect(reader.snapshot().task("app.done")).toBe(Done); + unsubscribe(); + await host.close(context); + }); + + it("wakes a Harness's scheduler: a task blocked on its missing definition runs once a contribution brings it", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serve) }], context); + const harness = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: registryReader(await host.registry(Extensions, context), context) }, + context, + ); + harness.resume(); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Done, undefined, { ownership: { kind: "conversation" } }), + context, + ); + const done = harness.waitForTask(id, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + await (await host.get(Plugin, {}, context)).install(context); + expect((await done).state.outcome).toEqual({ status: "completed", result: null }); + await harness.close(context); + await host.close(context); + }); +}); + +/** p3's pattern for Durable's views from other objects: the object that opens Durable contributes its Harness. */ +const Harnesses = defineRegistry, Harness | undefined>("app.harnesses", { version: 1 }); + +/** Opens a Harness (a new one per attempt), contributes it, and follows its task graph: its Session closing fails it. */ +const Durable = local("app.durable", { version: 1, params: {}, members: { close: method(), write: method(str()) } }); + +/** Follows the root conversation's entry kinds through the contributed Harness, from another object. */ +const Viewer = local("app.viewer", { version: 1, params: {}, members: { kinds: arr(str()) } }); + +function serveViews(host: FacetHost): void { + host.serve(Harnesses, { compose: (table) => table.at(-1)?.value.value }); + host.serve(Durable, (_, self) => { + const harness = self.open( + async (open) => { + const opened = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry: createRegistry() }, + open, + ); + await opened.root(open); + return opened; + }, + (h, _reason, close) => h.close(close), + ); + self.contribute(Harnesses, harness); + self.follow((observer, ctx) => harness.value.observeTaskGraph(observer, ctx)); + return { + close: async (ctx) => { + await harness.value.close(ctx); + return null; + }, + write: async (kind, ctx) => { + const root = await harness.value.conversation(ROOT_CONVERSATION_ID, ctx); + await root!.submit({ type: "write", entry: { kind } }, ctx); + return null; + }, + }; + }); + host.serve(Viewer, (_, self) => { + const harness = self.registry(Harnesses); + self.retryOn(harness); + const view = self.follow(async (observer, ctx) => + harness.value === undefined + ? undefined + : harness.value.observeConversation(ROOT_CONVERSATION_ID, observer, ctx), + ); + return { kinds: self.derive([view], ([v]) => v.entries.map((entry) => entry.kind)) }; + }); +} + +describe("Durable's views from other objects (slice 8 §5, S1)", () => { + it("a follower of a contributed Harness fails when that Harness's Session closes, and follows the next one", async () => { + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@app/core", facet: defineFacet(serveViews) }], context); + const durable = host.hold(Durable, {}, context); + const viewer = host.hold(Viewer, {}, context); + await new Promise((resolve) => setTimeout(resolve, 10)); + await durable.write("app.first", context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.kinds.value).toContain("app.first"); + // The Session closes: both followers fail `transient` (S1); a retry opens the next Harness, whose contribution + // wakes the viewer (`retryOn`), which follows it. + // The call is cut off by its object's failure: the Session's close fails the object that opened it. + await expect(durable.close(context)).rejects.toThrow("session closed"); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.state.value).toMatchObject({ kind: "failed", cause: { kind: "transient" } }); + durable.retry(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(viewer.state.value).toEqual({ kind: "live" }); + expect(viewer.kinds.value).not.toContain("app.first"); + await durable.write("app.second", context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(viewer.kinds.value).toContain("app.second"); + await host.close(context); + }); +}); diff --git a/packages/durable/test/harness-registry.test.ts b/packages/durable/test/harness-registry.test.ts index af852b0c1..b51002576 100644 --- a/packages/durable/test/harness-registry.test.ts +++ b/packages/durable/test/harness-registry.test.ts @@ -1,4 +1,3 @@ -import { Type } from "@earendil-works/pi-ai"; import { type AgentState, CompactionTask, @@ -15,6 +14,7 @@ import { wrapSection, wrapTool, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { agentHooks, resolveAgent, resolveSettings } from "../src/harness/agent.ts"; import { context } from "./session-support.ts"; @@ -25,7 +25,7 @@ function tool(name: string, extra: Partial = {}): AppTool { return { name, description: `${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), ...extra, }; diff --git a/packages/durable/test/harness-reports.test.ts b/packages/durable/test/harness-reports.test.ts index 623fafcdb..97a7acf51 100644 --- a/packages/durable/test/harness-reports.test.ts +++ b/packages/durable/test/harness-reports.test.ts @@ -1,4 +1,4 @@ -import { createModels, fauxAssistantMessage, Type } from "@earendil-works/pi-ai"; +import { createModels, fauxAssistantMessage } from "@earendil-works/pi-ai"; import { type AnyTask, createRegistry, @@ -14,6 +14,7 @@ import { section, wrapTool, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup } from "./chat-support.ts"; import { ControlledStorage, context } from "./session-support.ts"; @@ -189,7 +190,7 @@ describe("reports", () => { const echo = defineTool({ name: "echo", description: "echo", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }); registry.install(defineExtension({ name: "tools", tools: [echo] })); diff --git a/packages/durable/test/harness-restart-failure.test.ts b/packages/durable/test/harness-restart-failure.test.ts index 356f2318e..c8ece30c7 100644 --- a/packages/durable/test/harness-restart-failure.test.ts +++ b/packages/durable/test/harness-restart-failure.test.ts @@ -1,6 +1,7 @@ // What a crash (a host that dies, is evicted, or is reset) does to work that was running: a generation's interrupted // request is a failed attempt under the retry policy, and every other task ends visibly once it was cut off // `settings.restartLimit` times in a row (#11139). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -13,7 +14,6 @@ import { type Message, type Models, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -27,8 +27,10 @@ import { type SubmissionId, type TaskId, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf, unanswered, waitFor } from "./chat-support.ts"; import { addSection, addTool } from "./harness-support.ts"; import { context, followCommits } from "./session-support.ts"; @@ -281,7 +283,7 @@ describe("a generation interrupted while the model answers (#11139)", () => { defineTool({ name: "hold", description: "Waits", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async () => { started.resolve(); @@ -374,7 +376,7 @@ describe("other tasks cut off restartLimit times in a row (#11139)", () => { defineTool({ name: "heavy", description: "Never finishes", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async (_args, api, ctx) => { runs++; @@ -650,7 +652,7 @@ describe("crash edges (#11139)", () => { // As if the charge had been cut off twice before: the open after this crash counts 3, past the limit of 2. const raw = await openNodeSqliteStorage(path); const record = await raw.task(taskId, context); - await raw.commit([{ type: "task", value: { ...record!, restarts: 2 } }], () => {}, context); + await commitWrites(raw, [{ type: "task", value: { ...record!, restarts: 2 } }], context); await raw.close(context); const setup = chatSetup(); setup.settings.restartLimit = 2; @@ -750,7 +752,7 @@ describe("the restart limit on the normal path (#11139)", () => { defineTool({ name: "t", description: "t", - parameters: Type.Object({}), + parameters: obj({}), replay: "safe", execute: async () => ({}), }), @@ -770,8 +772,9 @@ describe("the restart limit on the normal path (#11139)", () => { for (let round = 0; round < rounds; round++) { await (await root.submit({ type: "input", content: `round ${round}` }, context)).wait(context); } - // One generation round with one tool call, as before the restart limit applied to every kind (#11327: 49). - expect(resolutions / rounds).toBe(49); + // One generation round with one tool call, no more often than before the restart limit applied to every kind + // (#11327: 49; group commit coalesces some reconcile passes). + expect(resolutions / rounds).toBeLessThanOrEqual(49); await harness.close(context); }); }); diff --git a/packages/durable/test/harness-run-failure.test.ts b/packages/durable/test/harness-run-failure.test.ts index ded8c8ebc..638a4715b 100644 --- a/packages/durable/test/harness-run-failure.test.ts +++ b/packages/durable/test/harness-run-failure.test.ts @@ -1,5 +1,6 @@ // `pi.live.runFailure`: why the newest run ended without an answer, as state a renderer shows (#11327). -import { type FauxResponseStep, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; + +import { type FauxResponseStep, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type AfterFailedRun, type Conversation, @@ -10,6 +11,7 @@ import { MemoryStorage, type RunFailure, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { chatSetup, openChat, unanswered } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -104,7 +106,7 @@ describe("pi.live.runFailure", () => { defineTool({ name: "hold", description: "Waits", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { started.resolve(); await gate.promise; diff --git a/packages/durable/test/harness-scheduler-indexes.test.ts b/packages/durable/test/harness-scheduler-indexes.test.ts index c42efd96e..33fd9ef88 100644 --- a/packages/durable/test/harness-scheduler-indexes.test.ts +++ b/packages/durable/test/harness-scheduler-indexes.test.ts @@ -3,7 +3,14 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createSession, defineTask, type Harness, type TaskId, type TaskRuntime } from "@earendil-works/pi-durable"; +import { + createSession, + defineTask, + type Harness, + type Storage, + type TaskId, + type TaskRuntime, +} from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { context } from "./session-support.ts"; @@ -104,16 +111,16 @@ describe("scheduler indexes", () => { }, context); await session.close(context); // Record every mark of the leaf, in commit order. - const storage = await openNodeSqliteStorage(path); + const storage: Storage = await openNodeSqliteStorage(path); const reasons: (string | undefined)[] = []; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - for (const write of writes) { + storage.commit = (batches, receive, commitContext) => { + for (const write of batches.flat()) { if (write.type === "task" && write.value.id === ids.leaf && write.value.abortRequested) { reasons.push(write.value.abortReason); } } - return commit(writes, adopt, commitContext); + return commit(batches, receive, commitContext); }; const { harness } = await openTasks(storage, [Hold]); harness.resume(); @@ -123,17 +130,20 @@ describe("scheduler indexes", () => { }); it("finalizes a chain of held owners in the commit that frees the last one", async () => { - const storage = await openNodeSqliteStorage(await sqlitePath()); + const storage: Storage = await openNodeSqliteStorage(await sqlitePath()); // The commit, by count, that wrote each terminal record. const endedIn = new Map(); let commits = 0; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; - for (const write of writes) { - if (write.type === "task" && write.value.state.status === "terminal") endedIn.set(write.value.id, commits); + storage.commit = (batches, receive, commitContext) => { + for (const writes of batches) { + commits++; + for (const write of writes) { + if (write.type === "task" && write.value.state.status === "terminal") + endedIn.set(write.value.id, commits); + } } - return commit(writes, adopt, commitContext); + return commit(batches, receive, commitContext); }; const opened = await openTasks(storage, [Hold]); const harness = opened.harness; diff --git a/packages/durable/test/harness-scheduler-retention.test.ts b/packages/durable/test/harness-scheduler-retention.test.ts index 918ffaa24..c50ba6fe6 100644 --- a/packages/durable/test/harness-scheduler-retention.test.ts +++ b/packages/durable/test/harness-scheduler-retention.test.ts @@ -125,21 +125,6 @@ async function sqlitePath(): Promise { } /** Count the task and conversation reads a storage serves. */ -function countReads(storage: Storage): { tasks: number; conversations: number } { - const reads = { tasks: 0, conversations: 0 }; - const task = storage.task.bind(storage); - const conversation = storage.conversation.bind(storage); - storage.task = (id, readContext) => { - reads.tasks++; - return task(id, readContext); - }; - storage.conversation = (id, readContext) => { - reads.conversations++; - return conversation(id, readContext); - }; - return reads; -} - async function open(storage?: Storage) { const opened = await openTasks(storage ?? (await openNodeSqliteStorage(await sqlitePath())), [ Hold, @@ -191,9 +176,7 @@ describe("scheduler retention", () => { }); it("holds a live owner for work later created in a forgotten conversation, loading its chain again", async () => { - const storage = await openNodeSqliteStorage(await sqlitePath()); - const reads = countReads(storage); - const { harness, root } = await open(storage); + const { harness, root } = await open(); // Live P -> ended B -> ended A -> conversation C, empty: forgotten. const tree = await root.commit(async (tx) => { const p = await tx.createTask(Hold, { name: "p" }, { ownership: { kind: "conversation" } }); @@ -206,15 +189,14 @@ describe("scheduler retention", () => { gate("b").resolve(); await harness.waitForTask(tree.b, context); const before = sizes(harness); - const loaded = reads.tasks + reads.conversations; const c = (await harness.conversation(tree.c, context))!; const d = await c.commit( (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), context, ); gate("p").resolve(); + // P holds for D: the scheduler loaded the chain from C up to P again. await eventually(async () => (await status(harness, tree.p)).status === "completing"); - expect(reads.tasks + reads.conversations).toBeGreaterThan(loaded); gate("d").resolve(); expect((await harness.waitForTask(d, context)).state.outcome.status).toBe("completed"); expect((await harness.waitForTask(tree.p, context)).state.outcome.status).toBe("completed"); @@ -307,7 +289,7 @@ describe("scheduler retention", () => { await harness.close(context); }); - it("keeps a scope busy while the chain of new work below a forgotten conversation loads", async () => { + it("loads the chain of new work below a forgotten conversation before admitting it, and keeps the scope busy", async () => { const path = await sqlitePath(); const storage = await openNodeSqliteStorage(path); const { harness, root } = await open(storage); @@ -318,28 +300,36 @@ describe("scheduler retention", () => { }, context); gate("b").resolve(); await harness.waitForTask(tree.b, context); - // Hold the chain load's read of C: the second read once holding, after the creating commit's own. - const read = storage.conversation.bind(storage); + await harness.close(context); + // Reopen, so that B is in no cache: the chain load reads it from Storage. Hold that read. + const reopened = await openNodeSqliteStorage(path); + const read = reopened.task.bind(reopened); const held = deferred(); const reached = deferred(); let holding = false; - let readsOfC = 0; - storage.conversation = async (id, readContext) => { - if (holding && id === tree.c && ++readsOfC === 2) { + reopened.task = async (id, readContext) => { + if (holding && id === tree.b) { reached.resolve(); await held.promise; } return read(id, readContext); }; - const c = (await harness.conversation(tree.c, context))!; + const second = (await open(reopened)).harness; + const c = (await second.conversation(tree.c, context))!; holding = true; - await c.commit((tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), context); + const committing = c.commit( + (tx) => tx.createTask(Hold, { name: "d" }, { ownership: { kind: "conversation" } }), + context, + ); + // Its admission loads the chain, for the memberships it writes: the commit waits for the read. await reached.promise; - expect(await settled(harness.waitForIdle(context))).toBe(false); + expect(await settled(committing)).toBe(false); held.resolve(); + await committing; + expect(await settled(second.waitForIdle(context))).toBe(false); gate("d").resolve(); - await harness.waitForIdle(context); - await harness.close(context); + await second.waitForIdle(context); + await second.close(context); }); it("keeps a chain when one commit ends its last task and creates new work below it", async () => { diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts index 4e066b77f..58a5543b4 100644 --- a/packages/durable/test/harness-storage-failure.test.ts +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -419,6 +419,8 @@ describe("a failed Storage call", () => { } } const { harness, root } = await started(new CancelledRead()); + // An entry to scan: the Session's cache answers the bounds of the context, not its entries. + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); const cancelling: Context = { ...context, abortSignal: controller.signal }; await expect(root.context(cancelling)).rejects.toBe(cancelled); await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index 829932323..5ef6f75ba 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -21,6 +20,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { nul, num, obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createBashTool } from "../src/tools/index.ts"; @@ -28,14 +28,14 @@ import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support. import { addHooks, addTool } from "./harness-support.ts"; import { context } from "./session-support.ts"; -const Count = Type.Object({ count: Type.Number() }); +const Count = obj({ count: num() }); /** A tool that returns `result`, with `extra` such as a schema. */ function returning(name: string, result: ToolExecutionResult, extra: Partial = {}): ToolRegistration { return defineTool({ name, description: name, - parameters: Type.Object({}), + parameters: obj({}), execute: async () => result, ...extra, }); @@ -55,7 +55,7 @@ async function nested(setup: ChatSetup, name: string): Promise { received = await api.executeTool(name, {}, callContext); return {}; @@ -132,17 +132,14 @@ describe("structured output", () => { it("stores exactly what callers and events get, without the model's output, and keeps a stored null", async () => { const setup = chatSetup(); addTool(setup.registry, returning("plain", { output: [{ type: "text", text: "a" }] })); - addTool( - setup.registry, - returning("nothing", { structuredOutput: null }, { structuredOutputSchema: Type.Null() }), - ); + addTool(setup.registry, returning("nothing", { structuredOutput: null }, { structuredOutputSchema: nul() })); const received: NestedToolExecutionResult[] = []; addTool( setup.registry, defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { received.push(await api.executeTool("plain", {}, callContext)); received.push(await api.executeTool("nothing", {}, callContext)); @@ -272,7 +269,7 @@ describe("structured output", () => { defineTool({ name: "throws", description: "throws", - parameters: Type.Object({}), + parameters: obj({}), structuredOutputSchema: Count, execute: async () => { throw new Error("permission denied"); @@ -344,7 +341,7 @@ describe("structured output", () => { defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { plain = await api.executeTool("plain", {}, callContext); counter = await api.executeTool("counter", {}, callContext); @@ -375,7 +372,7 @@ describe("structured output", () => { defineTool({ name: "probe", description: "probe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { received.push(await api.executeTool("bash", { command: "printf ok" }, callContext)); received.push(await api.executeTool("bash", { command: "printf bad; exit 4" }, callContext)); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index ac7cc9b55..26c7d7fb6 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -8,7 +8,6 @@ import { fauxToolCall, type Models, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -26,6 +25,7 @@ import { type NextTaskState, type Registry, type Storage, + type StorageWrite, type TaskGraph, type TaskId, type TaskOptions, @@ -34,12 +34,13 @@ import { ToolResultEntry, type Tx, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; import { generations, graphHistory, slotsOf, viewHistory } from "./live-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, refusingAt } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; // ─── A scriptable task ────────────────────────────────────────────────────── @@ -1191,7 +1192,7 @@ function blockingTool(name: string): { started: Promise; registration: Too registration: defineTool({ name, description: `The ${name} tool`, - parameters: Type.Object({}), + parameters: obj({}), executionMode: "sequential", execute: async (_args, api, callContext) => { api.output("partial"); @@ -1217,7 +1218,7 @@ describe("tool rounds", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1297,7 +1298,7 @@ describe("tool rounds", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1361,19 +1362,20 @@ describe("tool rounds", () => { let reject: TaskId | undefined; const failure = new Error("disk gone"); class Failing extends ControlledStorage { - override async commit( - writes: Parameters[0], - adopt: Parameters[1], + override commit( + batches: Parameters[0], + receive: Parameters[1], ctx: Parameters[2], ) { - const finalizes = writes.some( - (write) => write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", - ); - if (finalizes) { - reject = undefined; - throw failure; - } - return super.commit(writes, adopt, ctx); + const refuses = (writes: readonly StorageWrite[]): boolean => { + const finalizes = writes.some( + (write) => + write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", + ); + if (finalizes) reject = undefined; + return finalizes; + }; + return refusingAt((...args) => super.commit(...args), batches, receive, ctx, refuses, failure); } } const storage = new Failing(); @@ -1647,19 +1649,20 @@ describe("conversation abort and boundaries", () => { let reject: TaskId | undefined; const failure = new Error("disk gone"); class Failing extends ControlledStorage { - override async commit( - writes: Parameters[0], - adopt: Parameters[1], + override commit( + batches: Parameters[0], + receive: Parameters[1], ctx: Parameters[2], ) { - const finalizes = writes.some( - (write) => write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", - ); - if (finalizes) { - reject = undefined; - throw failure; - } - return super.commit(writes, adopt, ctx); + const refuses = (writes: readonly StorageWrite[]): boolean => { + const finalizes = writes.some( + (write) => + write.type === "task" && write.value.id === reject && write.value.state.status === "terminal", + ); + if (finalizes) reject = undefined; + return finalizes; + }; + return refusingAt((...args) => super.commit(...args), batches, receive, ctx, refuses, failure); } } const storage = new Failing(); @@ -1823,7 +1826,7 @@ describe("tool rounds as the view shows them", () => { defineTool({ name: "noop", description: "Does nothing", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [] }), }), ); @@ -1863,7 +1866,7 @@ describe("tool rounds as the view shows them", () => { defineTool({ name: "delegate", description: "Starts work in a conversation it owns", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { await api.commit(async (tx) => { const child = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); @@ -1903,7 +1906,7 @@ describe("tool rounds as the view shows them", () => { defineTool({ name: "broken", description: "Starts owned work, then returns a result that is not strict JSON", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { await api.commit((tx) => tx.createTask(Node, { name: "held" }, owned(api.taskId)), callContext); return { output: [], details: { fn: (() => 1) as unknown as JsonValue } }; diff --git a/packages/durable/test/harness-submissions.test.ts b/packages/durable/test/harness-submissions.test.ts index 08630183a..5ee8eac22 100644 --- a/packages/durable/test/harness-submissions.test.ts +++ b/packages/durable/test/harness-submissions.test.ts @@ -163,9 +163,14 @@ describe("submissions", () => { } } const setup = chatSetup(); - setup.faux.setResponses([unanswered().step]); - const { harness, root } = await openChat(new HeldReads(), setup); - const submission = await root.submit({ type: "input", content: "hi" }, context); + setup.faux.setResponses([unanswered().step, unanswered().step]); + const storage = new HeldReads(); + const first = await openChat(storage, setup); + const { id } = await first.root.submit({ type: "input", content: "hi" }, context); + await first.harness.close(context); + // Reopen: the Session caches the submissions it writes, so only a reopened one reads it from Storage. + const { harness } = await openChat(storage.reopen(), setup); + const submission = (await harness.submission(id, context))!; hold = true; const waiting = submission.wait(context); await entered.promise; diff --git a/packages/durable/test/harness-submit-in-commit.test.ts b/packages/durable/test/harness-submit-in-commit.test.ts index 3d8ad2986..9f4e08d9e 100644 --- a/packages/durable/test/harness-submit-in-commit.test.ts +++ b/packages/durable/test/harness-submit-in-commit.test.ts @@ -1,5 +1,6 @@ // Admission for hosts that must not start runs: `whenIdle: "reject"`, and `submit(tx, draft)` inside the caller's // commit (#11149). + import { withTelemetry } from "@earendil-works/chord/context"; import { type AssistantMessage, @@ -7,7 +8,6 @@ import { fauxAssistantMessage, fauxText, fauxToolCall, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -26,6 +26,7 @@ import { type Tx, UserEntry, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat, unanswered, waitFor } from "./chat-support.ts"; @@ -80,7 +81,7 @@ function addHold(setup: ChatSetup) { defineTool({ name: "hold", description: "Holds until released", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { reached.resolve(); await release.promise; @@ -180,7 +181,7 @@ describe("whenIdle reject", () => { defineTool({ name: "nudge", description: "Steers another conversation", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { const handle = (await api.conversation(other!.id, callContext))!; const draft = { type: "input", content: "nudge", whenBusy: "steer", whenIdle: "reject" } as const; @@ -525,7 +526,7 @@ describe("submit inside the caller's commit", () => { defineTool({ name: "delegate", description: "Delegates to another conversation", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { toolTask = api.taskId; const handle = (await api.conversation(other!.id, callContext))!; @@ -611,7 +612,7 @@ describe("submit inside the caller's commit", () => { defineTool({ name: "hold", description: "Holds until released, also when aborted", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, callContext) => { handle = (await api.conversation(other!.id, callContext))!; callContext.abortSignal!.addEventListener("abort", () => abortSeen.resolve(), { once: true }); diff --git a/packages/durable/test/harness-support.ts b/packages/durable/test/harness-support.ts index 499f81846..d2c443c45 100644 --- a/packages/durable/test/harness-support.ts +++ b/packages/durable/test/harness-support.ts @@ -7,7 +7,7 @@ import type { ToolResultMessage, UserMessage, } from "@earendil-works/pi-ai"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { type AnyTask, createRegistry, @@ -23,10 +23,11 @@ import { section, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { context } from "./session-support.ts"; export function tool(name: string, description = `${name} tool`): ToolRegistration { - return defineTool({ name, description, parameters: Type.Object({}), execute: async () => ({ output: [] }) }); + return defineTool({ name, description, parameters: obj({}), execute: async () => ({ output: [] }) }); } /** Open a Harness with a fresh registry holding the named tools. */ diff --git a/packages/durable/test/harness-task-graph.test.ts b/packages/durable/test/harness-task-graph.test.ts index c2d63694b..a23a95d49 100644 --- a/packages/durable/test/harness-task-graph.test.ts +++ b/packages/durable/test/harness-task-graph.test.ts @@ -365,7 +365,7 @@ describe("task graph view", () => { await harness.close(context); }); - it("advances with the conversation view inside the one storage commit that changes both", async () => { + it("advances with the conversation view at the publication of the one commit that changes both", async () => { const storage = new ControlledStorage(); const { Parent, Child } = family(gatesNever()); const { harness } = await openTasks(storage, [Parent, Child]); @@ -383,8 +383,8 @@ describe("task graph view", () => { await tx.appendEntry(root.id, { kind: "note" }); await tx.createTask(Parent, null, { ownership: { kind: "conversation" } }); }, context); - // Both inside the one adopt, in no promised order. - expect([...seen].sort()).toEqual(["graph adopting=true", "view adopting=true"]); + // Both at the one publication, after Storage stored it, in no promised order. + expect([...seen].sort()).toEqual(["graph adopting=false", "view adopting=false"]); view.detach(); graph.detach(); await harness.close(context); diff --git a/packages/durable/test/harness-tasks-recovery.test.ts b/packages/durable/test/harness-tasks-recovery.test.ts index 714fdc202..07fcd7f27 100644 --- a/packages/durable/test/harness-tasks-recovery.test.ts +++ b/packages/durable/test/harness-tasks-recovery.test.ts @@ -2,7 +2,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { Context, JsonValue } from "@earendil-works/chord"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { createRegistry, defineDoc, @@ -15,6 +15,7 @@ import { type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { addTask, addTool } from "./harness-support.ts"; @@ -482,7 +483,7 @@ describe("blocked tasks", () => { defineTool({ name: "unrelated", description: "unrelated", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }), ); diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index 5cce1635b..17bfe604f 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -1,6 +1,6 @@ import type { Context } from "@earendil-works/chord"; import { withAbortSignal, withCancel } from "@earendil-works/chord/context"; -import { createModels, Type } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai"; import { type CommitPublication, type Conversation, @@ -18,8 +18,10 @@ import { type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; +import type { SessionImpl } from "../src/session/session.ts"; import { addHooks, addTask, addTool, user } from "./harness-support.ts"; import { ControlledStorage, context, flush, followCommits, LateResolvingStorage, onValue } from "./session-support.ts"; import { aborted, abortedWith, completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; @@ -85,6 +87,20 @@ function withSignal(signal: AbortSignal): Context { return withAbortSignal(signal, context); } +/** Hold the Session line with a commit whose callback waits: the jobs queued after it start once it is released. */ +function holdLine(harness: { commit(change: () => Promise, context: Context): Promise }): { + readonly entered: Promise; + release(): void; +} { + const entered = deferred(); + const release = deferred(); + void harness.commit(async () => { + entered.resolve(); + await release.promise; + }, context); + return { entered: entered.promise, release: () => release.resolve() }; +} + describe("task phases", () => { it("continues one invocation through checkpoint progress and completes with a typed result", async () => { const seen: number[] = []; @@ -235,7 +251,15 @@ describe("task phases", () => { }, abort: async () => {}, }); - const { harness, root } = await openRoot([Writer, Child]); + class CountingStorage extends MemoryStorage { + documentScans = 0; + override scanDocuments(...args: Parameters) { + this.documentScans++; + return super.scanDocuments(...args); + } + } + const storage = new CountingStorage(); + const { harness, root } = await openRoot([Writer, Child], { storage }); const conversation = await harness.createConversation({ ownership: { kind: "ownerless" } }, context); const id = await conversation.commit( (tx) => tx.createTask(Writer, null, { ownership: { kind: "conversation" } }), @@ -252,6 +276,10 @@ describe("task phases", () => { expect(entry?.conversationId).toBe(conversation.id); expect(progressSeen).toEqual(["wrote"]); expect(await harness.snapshot(Progress, id, context)).toBeUndefined(); + // The Session indexed the documents of the tasks it created, so no terminal commit scanned for them, and it dropped + // each task's entry at its end. + expect(storage.documentScans).toBe(0); + expect((harness as unknown as SessionImpl).documentCacheSizes.taskDocuments).toBe(0); expect(await harness.getTask(id, context)).toEqual(receipt); expect((await harness.waitForTask(childId!, context)).conversationId).toBe(conversation.id); expect(root.id).not.toBe(conversation.id); @@ -397,7 +425,7 @@ describe("task phases", () => { defineTool({ name: "late", description: "late", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({}), }), ); @@ -1208,13 +1236,11 @@ describe("task close", () => { it("writes no fault when close seals while the step after a failed phase is queued", async () => { const storage = new ControlledStorage(); - let held: ReturnType | undefined; + let held: ReturnType | undefined; let harnessRef: Awaited>["harness"] | undefined; - const Throws = oneStep("test.close-fault", async (_task, runtime) => { - held = storage.holdCommits(); - void harnessRef!.commit(async (tx) => { - await tx.appendEntry(runtime.conversationId, { kind: "blocker" }); - }, context); + const Throws = oneStep("test.close-fault", async () => { + // The step after this phase queues behind a job that holds the line. + held = holdLine(harnessRef!); throw new Error("would fault"); }); const { harness, root } = await openRoot([Throws], { storage }); @@ -1269,14 +1295,11 @@ describe("task close", () => { it("rejects a runtime commit that was queued on the line when close sealed it", async () => { const storage = new ControlledStorage(); let queued: Promise | undefined; - let held: ReturnType | undefined; + let held: ReturnType | undefined; let harnessRef: Awaited>["harness"] | undefined; const proceed = deferred(); const Queued = oneStep("test.close-queued-commit", async (_task, runtime) => { - held = storage.holdCommits(); - void harnessRef!.commit(async (tx) => { - await tx.appendEntry(runtime.conversationId, { kind: "blocker" }); - }, context); + held = holdLine(harnessRef!); queued = runtime.commit(() => completed(null), context).catch((error: unknown) => error); // Ignores the close signal, so the invocation is still alive when the queued commit reaches the line. await proceed.promise; @@ -1459,7 +1482,7 @@ describe("publication at adoption", () => { }); } - it("resolves a waitForIdle() called after adoption, before the storage resolved, after host listeners", async () => { + it("resolves a waitForIdle() called while the storage stores the commit at its publication, after host listeners", async () => { const storage = new LateResolvingStorage(); const gate = deferred(); const Gated = gated("test.gap-idle", gate.promise); @@ -1484,7 +1507,7 @@ describe("publication at adoption", () => { gate.resolve(); await gapped.promise; await idle; - expect(order).toEqual(["host", "idle committing=true"]); + expect(order).toEqual(["host", "idle committing=false"]); await harness.close(context); }); @@ -1545,7 +1568,7 @@ describe("publication at adoption", () => { await closing; }); - it("keeps a task wait resolved when the Harness closes after adoption, before the storage resolved", async () => { + it("ends a task wait when the Harness closes while the terminal commit is stored, and still stores it", async () => { const storage = new LateResolvingStorage(); const gate = deferred(); const Gated = gated("test.close-in-gap", gate.promise); @@ -1567,8 +1590,12 @@ describe("publication at adoption", () => { gate.resolve(); await gapped.promise; await closing; - expect(await waited).toEqual({ status: "completed", result: null }); + // Close ends the waits before it drains the commit in flight, which is published and stored all the same. + expect(await waited).toMatchObject({ message: expect.stringContaining("closed") }); expect(await harness.closed).toEqual({ kind: "closed" }); + const reopened = await openRoot([Gated], { storage: storage.reopen() }); + expect((await reopened.harness.getTask(id, context))?.state.status).toBe("terminal"); + await reopened.harness.close(context); }); it("fails the Session with the storage error when it rejects after adopting, and reports an observer's throw", async () => { @@ -1627,14 +1654,22 @@ describe("publication at adoption", () => { await harness.close(context); }); - it("calls a host listener added on a Harness between adopt and the dispatch only when one was registered at adopt", async () => { + it("calls a host listener added on a Harness during a publication only when one was registered at it", async () => { const storage = new ControlledStorage(); const { harness, root } = await openRoot([], { storage }); const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); const late: number[] = []; let stopLate = { detach: (): void => {} }; - // No host listener at adopt, although the Harness's own observers run: no dispatch, so the late one misses it. - storage.afterAdopt = () => { + // Runs once, during the next publication, after its dispatch was queued or not. + let during: (() => void) | undefined; + followCommits(harness, () => { + const run = during; + during = undefined; + run?.(); + }); + // No host listener at the publication, although the Harness's own observers run: no dispatch, so the late one + // misses it. + during = () => { stopLate = harness.observeCommits({ advance: (publication) => { late.push(publication.seq); @@ -1651,7 +1686,7 @@ describe("publication at adoption", () => { // One registered at adopt: the dispatch reads the listeners when it runs, the late one included. harness.observeCommits({ advance: () => {}, end: () => {} }); const second: number[] = []; - storage.afterAdopt = () => { + during = () => { harness.observeCommits({ advance: (publication) => { second.push(publication.seq); diff --git a/packages/durable/test/harness-telemetry.test.ts b/packages/durable/test/harness-telemetry.test.ts index 59537179e..58bcbc9e8 100644 --- a/packages/durable/test/harness-telemetry.test.ts +++ b/packages/durable/test/harness-telemetry.test.ts @@ -1,5 +1,5 @@ import { withTelemetry } from "@earendil-works/chord/context"; -import { fauxAssistantMessage, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai"; import { CodemodeSandbox } from "@earendil-works/pi-codemode"; import { CompactionTask, @@ -11,6 +11,7 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { InMemoryTelemetryContext, NOOP_TELEMETRY_CONTEXT, @@ -53,7 +54,7 @@ describe("Durable telemetry", () => { defineTool({ name: "inner", description: "inner", - parameters: Type.Object({ secret: Type.String() }), + parameters: obj({ secret: str() }), execute: async () => ({ output: [{ type: "text", text: "private-result" }] }), }), ); @@ -62,7 +63,7 @@ describe("Durable telemetry", () => { defineTool({ name: "outer", description: "outer", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, ctx) => { await api.executeTool("inner", { secret: "private-argument" }, ctx); return { output: [{ type: "text", text: "private-output" }] }; @@ -347,7 +348,7 @@ describe("Durable telemetry", () => { defineTool({ name: "hold", description: "hold", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { reached.resolve(); await gate.promise; @@ -415,7 +416,7 @@ describe("Durable telemetry", () => { defineTool({ name: "delegate", description: "delegate", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api, ctx) => { const child = await api.commit( (tx) => tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }), @@ -952,7 +953,7 @@ describe("Durable telemetry", () => { defineTool({ name: "once", description: "once", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { runs++; return { output: [] }; @@ -965,7 +966,7 @@ describe("Durable telemetry", () => { name: "hold", description: "hold", replay: "safe", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, _api, ctx) => { if (!recover) { reached.resolve(); diff --git a/packages/durable/test/harness-tool-hooks.test.ts b/packages/durable/test/harness-tool-hooks.test.ts index 1847c768b..beca471e1 100644 --- a/packages/durable/test/harness-tool-hooks.test.ts +++ b/packages/durable/test/harness-tool-hooks.test.ts @@ -1,5 +1,6 @@ // `afterTool` runs on every result a tool task writes, also those the Harness decides without executing the tool // (#11043). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -10,7 +11,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -29,8 +29,10 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { bool, obj, opt, str } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addHooks, addTool } from "./harness-support.ts"; import { context } from "./session-support.ts"; @@ -78,7 +80,7 @@ function stored(): Promise { return new Promise((resolve) => setTimeout(resolve, 50)); } -const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +const Parameters = obj({ text: opt(str()) }); function tool( name: string, @@ -317,7 +319,7 @@ describe("afterTool on results the call phase decides", () => { addTool( setup.registry, tool("typed", async () => ({ output: [], structuredOutput: { ok: true } }), { - structuredOutputSchema: Type.Object({ ok: Type.Boolean() }), + structuredOutputSchema: obj({ ok: bool() }), }), ); const nested: NestedToolExecutionResult[] = []; @@ -763,7 +765,7 @@ describe("calls to tools the request did not offer", () => { if (version === 1) { const storage = await openNodeSqliteStorage(path); const record = (await storage.task(generation, context))!; - await storage.commit([{ type: "task", value: { ...record, version: 1 } }], () => {}, context); + await commitWrites(storage, [{ type: "task", value: { ...record, version: 1 } }], context); await storage.close(context); } diff --git a/packages/durable/test/harness-tools-recovery.test.ts b/packages/durable/test/harness-tools-recovery.test.ts index 31485ab25..cf050f8df 100644 --- a/packages/durable/test/harness-tools-recovery.test.ts +++ b/packages/durable/test/harness-tools-recovery.test.ts @@ -8,7 +8,6 @@ import { fauxText, fauxToolCall, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { defineTool, @@ -21,6 +20,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; @@ -55,7 +55,7 @@ function tool( execute: ToolRegistration["execute"], extra: Partial = {}, ): ToolRegistration { - return defineTool({ name, description: name, parameters: Type.Object({}), execute, ...extra }); + return defineTool({ name, description: name, parameters: obj({}), execute, ...extra }); } function call(name: string, id = "c1"): AssistantMessage { @@ -133,6 +133,70 @@ describe("tool recovery", () => { await opened.harness.close(context); }); + it("reruns a safe tool whose running slot a crash after its intent lost, or a sibling stored, and shows it running", async () => { + for (const sibling of [false, true]) { + const path = await sqlitePath(); + const setup = chatSetup(); + // The running slot is presentation, held in memory; no deadline stores it before the crash. + setup.settings.progress = { persistWithinMs: Number.POSITIVE_INFINITY }; + const started = deferred(); + let runs = 0; + let reopened: Harness | undefined; + const seen: (string | undefined)[] = []; + addTool( + setup.registry, + tool( + "work", + async (_args, api, callContext) => { + runs++; + if (runs === 1) { + started.resolve(); + await aborted(callContext.abortSignal!); + } + const slots = (await reopened!.snapshot(LiveDoc, 1 as never, callContext))?.tools ?? []; + seen.push(slots.find((slot) => slot.taskId === api.taskId)?.status); + return {}; + }, + { replay: "safe" }, + ), + ); + // A sibling that ends once `work` started: its result commit reads `pi.live`, which stores what is held there. + addTool( + setup.registry, + tool("quick", async () => { + await started.promise; + return {}; + }), + ); + const calls = [fauxToolCall("work", {}, { id: "w" })]; + if (sibling) calls.push(fauxToolCall("quick", {}, { id: "q" })); + setup.faux.setResponses([fauxAssistantMessage(calls, { stopReason: "toolUse" }), DONE]); + const crashed = await open(path, setup); + const id = (await crashed.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + const slot = async (harness: Harness, name: string) => + (await harness.snapshot(LiveDoc, crashed.root.id, context))?.tools?.find((each) => each.name === name); + const live = (harness: Harness) => slot(harness, "work"); + const workId = (await live(crashed.harness))!.taskId!; + if (sibling) await waitFor(async () => (await slot(crashed.harness, "quick"))?.status === "done"); + // The process dies here: another one opens the same database, without the held presentation. + const next = await openChat(await openNodeSqliteStorage(path), setup); + reopened = next.harness; + expect((await next.harness.getTask(workId, context))?.state).toMatchObject({ + checkpoint: { phase: "execute" }, + }); + expect((await live(next.harness))?.status).toBe(sibling ? "running" : "pending"); + next.harness.resume(); + expect((await (await next.harness.submission(id, context))!.wait(context)).status).toBe("done"); + // The rerun shows the slot running again. + expect(seen).toEqual(["running"]); + const [result] = results(await allEntries(next.root)).filter((message) => message.toolCallId === "w"); + expect(result!.isError).toBe(false); + await next.harness.close(context); + await crashed.harness.close(context); + } + }); + it("reruns a tool only when both the stored and the current replay policy are safe", async () => { const cases: [stored: "safe" | "unsafe", current: "safe" | "unsafe", reruns: boolean][] = [ ["safe", "safe", true], diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index 6849861b8..92aec1cf6 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -11,7 +11,6 @@ import { type Models, type SystemMessage, type ToolResultMessage, - Type, } from "@earendil-works/pi-ai"; import { AgentDoc, @@ -32,6 +31,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj, opt, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; @@ -40,7 +40,7 @@ import { addHooks, addTask, addTool, type Installed } from "./harness-support.ts import { context } from "./session-support.ts"; import { aborted, deferred } from "./task-support.ts"; -const EchoParameters = Type.Object({ text: Type.Optional(Type.String()) }); +const EchoParameters = obj({ text: opt(str()) }); type Echo = ToolRegistration; type Execute = Echo["execute"]; @@ -752,6 +752,75 @@ describe("generation hooks", () => { await harness.close(context); }); + it("chains beforeProviderRequest over every provider request body", async () => { + const setup = chatSetup(); + const seen: { hook: string; model: string; payload: unknown }[] = []; + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ model, payload }) => { + seen.push({ hook: "first", model: `${model.provider}/${model.id}`, payload }); + return { payload: { ...(payload as object), first: true } }; + }, + }); + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ model, payload }) => { + seen.push({ hook: "keep", model: `${model.provider}/${model.id}`, payload }); + return undefined; + }, + }); + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ payload }) => ({ payload: { ...(payload as object), last: true } }), + }); + const sent: unknown[] = []; + // pi-ai builds a body per provider call, and again when it resends within one attempt. + const respond: FauxResponseStep = async (_context, options, _state, model) => { + sent.push(await options!.onPayload!({ body: 1 }, model)); + sent.push(await options!.onPayload!({ body: 2 }, model)); + return DONE; + }; + const { harness, status } = await run(setup, [respond]); + expect(status).toBe("done"); + expect(sent).toEqual([ + { body: 1, first: true, last: true }, + { body: 2, first: true, last: true }, + ]); + expect(seen).toEqual([ + { hook: "first", model: "faux/faux-1", payload: { body: 1 } }, + { hook: "keep", model: "faux/faux-1", payload: { body: 1, first: true } }, + { hook: "first", model: "faux/faux-1", payload: { body: 2 } }, + { hook: "keep", model: "faux/faux-1", payload: { body: 2, first: true } }, + ]); + await harness.close(context); + }); + + it("reports a throwing beforeProviderRequest hook and continues the chain", async () => { + const setup = chatSetup(); + addHooks( + setup.registry, + GenerationTask, + { + beforeProviderRequest: () => { + throw new Error("broken hook"); + }, + }, + "broken", + ); + addHooks(setup.registry, GenerationTask, { + beforeProviderRequest: ({ payload }) => ({ payload: { ...(payload as object), after: true } }), + }); + let sent: unknown; + const respond: FauxResponseStep = async (_context, options, _state, model) => { + sent = await options!.onPayload!({ body: 1 }, model); + return DONE; + }; + const { harness, status } = await run(setup, [respond]); + expect(status).toBe("done"); + expect(sent).toEqual({ body: 1, after: true }); + expect(setup.infos.filter((info) => info.kind === "extension_error")).toEqual([ + expect.objectContaining({ extension: "broken" }), + ]); + await harness.close(context); + }); + it("keeps the run's input open across an onYield continuation and answers it with the final answer", async () => { const setup = chatSetup(); let yields = 0; diff --git a/packages/durable/test/harness-turn-limit.test.ts b/packages/durable/test/harness-turn-limit.test.ts index e5d819468..7f8ad61e1 100644 --- a/packages/durable/test/harness-turn-limit.test.ts +++ b/packages/durable/test/harness-turn-limit.test.ts @@ -1,8 +1,9 @@ // Per-conversation turn limits (`AgentState.maxTurns`) and continuing a run that reached one (#11043). + import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type AfterFailedRun, type Conversation, @@ -16,6 +17,7 @@ import { ToolResultEntry, ToolTask, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; @@ -33,7 +35,7 @@ afterEach(async () => { directories.clear(); }); -const Parameters = Type.Object({}); +const Parameters = obj({}); function tool( name: string, diff --git a/packages/durable/test/harness-view-run.test.ts b/packages/durable/test/harness-view-run.test.ts index 61431cd13..79f446098 100644 --- a/packages/durable/test/harness-view-run.test.ts +++ b/packages/durable/test/harness-view-run.test.ts @@ -1,5 +1,6 @@ // What a renderer sees of a run through the conversation view: the transcript window and `pi.live`, every change a // commit makes, rebuilt from operations alone. This replaced the event stream (`watchEvents`, #11327). + import { GCProfiler, getHeapStatistics, setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; import type { Draft } from "@earendil-works/chord"; @@ -11,7 +12,6 @@ import { fauxText, fauxThinking, fauxToolCall, - Type, } from "@earendil-works/pi-ai"; import { type Conversation, @@ -22,6 +22,7 @@ import { type LiveState, MemoryStorage, } from "@earendil-works/pi-durable"; +import { num, obj } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat, textOf, waitFor } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; @@ -126,7 +127,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "print", description: "Prints", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (_args, api) => { api.output("one\n"); await new Promise((resolve) => setTimeout(resolve, 150)); @@ -259,7 +260,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "tail", description: "Prints lines", - parameters: Type.Object({}), + parameters: obj({}), outputLimits: { maxLines: 3, retain: "tail" }, execute: async (_args, api) => { for (let line = 0; line < 6; line++) { @@ -339,7 +340,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "wait", description: "Waits until aborted", - parameters: Type.Object({}), + parameters: obj({}), execute: (_args, _api, callContext) => aborted(callContext.abortSignal!), }), ); @@ -380,7 +381,7 @@ describe("a run as the conversation view shows it", () => { defineTool({ name: "hold", description: "Waits", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => { await gate.promise; return {}; diff --git a/packages/durable/test/harness-view.test.ts b/packages/durable/test/harness-view.test.ts index 2f0913fc0..8e5aaa78a 100644 --- a/packages/durable/test/harness-view.test.ts +++ b/packages/durable/test/harness-view.test.ts @@ -142,7 +142,8 @@ describe("conversation view", () => { await submission.wait(context); await harness.waitForIdle(context); await drained(); - expect(frames).toHaveLength(touching.count); + // One frame per stored commit, and one for the attempt, a deferred commit stored with the answer. + expect(frames).toHaveLength(touching.count + 1); const last = replay(initial, frames); expect(last).toEqual(await committed(harness, root, initial.conversation, last.entries.length)); expect(frames[0]!.ops).toContainEqual(["p", ["entries"], 0, 0, [expect.objectContaining({ kind: "pi.user" })]]); diff --git a/packages/durable/test/indexeddb-storage.test.ts b/packages/durable/test/indexeddb-storage.test.ts index 352b7ca0e..85ba1751d 100644 --- a/packages/durable/test/indexeddb-storage.test.ts +++ b/packages/durable/test/indexeddb-storage.test.ts @@ -3,11 +3,13 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; -import { IDBKeyRange as FakeIDBKeyRange, IDBFactory } from "fake-indexeddb"; -import { describe, expect, it } from "vitest"; +import { IDBKeyRange as FakeIDBKeyRange, IDBObjectStore as FakeIDBObjectStore, IDBFactory } from "fake-indexeddb"; +import { describe, expect, it, vi } from "vitest"; import { idFromNumber } from "../src/ids.ts"; import { IndexedDbStorage } from "../src/storage/indexeddb.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import type { + CommitResult, ConversationId, DocumentCreate, DocumentId, @@ -22,9 +24,6 @@ import type { } from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - Object.defineProperty(globalThis, "IDBKeyRange", { configurable: true, value: FakeIDBKeyRange }); const context = BACKGROUND_CONTEXT; @@ -42,10 +41,14 @@ class ReopeningStorage implements Storage { this.indexedDB = indexedDB; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { - if (this.closed) return this.current.commit(writes, adopt, commitContext); + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.closed) return this.current.commit(batches, receive, commitContext); try { - return await this.current.commit(writes, adopt, commitContext); + return await this.current.commit(batches, receive, commitContext); } finally { await this.current.close(context); this.current = await IndexedDbStorage.open(this.name, { indexedDB: this.indexedDB }); @@ -135,7 +138,7 @@ function pendingTask(id: TaskId, conversationId = ROOT_CONVERSATION_I } async function createRoot(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); } it("persists minted IDs across reopen", async () => { @@ -164,15 +167,16 @@ it("rolls back table writes when a later document copy fails", async () => { history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record: source, content: { kind: "base", version: 1, value: { count: 1 } } }], - adopt, context, ); const taskId = await storage.mintId>(); const copyId = await storage.mintId(); await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", value: pendingTask(taskId) }, { @@ -181,7 +185,6 @@ it("rolls back table writes when a later document copy fails", async () => { source: { id: sourceId, at: "current" }, }, ], - adopt, context, ), ).rejects.toThrow("was rejected"); @@ -199,7 +202,8 @@ it("scans entries and head markers through fork ancestry", async () => { await createRoot(storage); const rootFirst = await storage.mintId(); const rootMarker = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: { id: rootFirst, conversationId: ROOT_CONVERSATION_ID, kind: "first" } }, { @@ -212,24 +216,23 @@ it("scans entries and head markers through fork ancestry", async () => { }, }, ], - adopt, context, ); const childId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: childId, parent: { conversationId: ROOT_CONVERSATION_ID, at: rootMarker } }, }, ], - adopt, context, ); const childEntry = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [{ type: "entry", value: { id: childEntry, conversationId: childId, kind: "child" } }], - adopt, context, ); expect( @@ -279,7 +282,7 @@ it("keeps one readwrite transaction active through many mixed writes", async () }, ); } - await storage.commit(writes, adopt, context); + await commitWrites(storage, writes, context); expect((await storage.entry(lastEntryId!, context))?.entry.data).toEqual({ index: 39 }); expect(await storage.task(lastTaskId!, context)).toEqual(pendingTask(lastTaskId!)); expect((await storage.submission(lastSubmissionId!, context))?.requestId).toBe("bulk-39"); @@ -288,3 +291,171 @@ it("keeps one readwrite transaction active through many mixed writes", async () await storage.close(context); } }); + +describe("commit", () => { + /** An entry of the root conversation. */ + const entryWrite = (id: EntryId): StorageWrite => ({ + type: "entry", + value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "batch" }, + }); + const sourceRecord = (id: DocumentId, conversationId: ConversationId = ROOT_CONVERSATION_ID): DocumentCreate => ({ + id, + kind: "source", + scope: { kind: "conversation", conversationId }, + history: "rewindable", + fork: "asOf", + }); + + /** Opens a storage with its root conversation, runs `test` on it, closes it. */ + async function withStorage(name: string, test: (storage: IndexedDbStorage) => Promise): Promise { + const storage = await IndexedDbStorage.open(name, { indexedDB: new IDBFactory() }); + try { + await createRoot(storage); + await test(storage); + } finally { + await storage.close(context); + } + } + + it("stores the batches before a refused one, and the storage goes on", () => + withStorage("batch-prefix", async (storage) => { + const first = await storage.mintId(); + const after = await storage.mintId(); + const receipts: [number, Seq][] = []; + const result = await storage.commit( + [[entryWrite(first)], [entryWrite(first)], [entryWrite(after)]], + (index, seq) => receipts.push([index, seq]), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: ID ${first} already belongs to entry`); + expect(receipts.map(([index]) => index)).toEqual([0]); + expect((await storage.entry(first, context))?.commitSeq).toBe(receipts[0]![1]); + expect(await storage.entry(after, context)).toBeUndefined(); + // The prefix advanced the metadata: the next commit takes the next seq. + const next = await commitWrites(storage, [entryWrite(after)], context); + expect(next).toBe(receipts[0]![1] + 1); + })); + + it("resolves a copy against the state an earlier batch of the call left", () => + withStorage("batch-copy", async (storage) => { + const sourceId = await storage.mintId(); + const childId = await storage.mintId(); + const copyId = await storage.mintId(); + const result = await storage.commit( + [ + [ + { + type: "document.create", + record: sourceRecord(sourceId), + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + [ + { + type: "document.change", + id: sourceId, + content: { kind: "delta", version: 1, ops: [["s", ["count"], 2]] }, + }, + ], + [ + { type: "conversation", value: { id: childId } }, + { + type: "document.copy", + record: sourceRecord(copyId, childId), + source: { id: sourceId, at: "current" }, + }, + ], + ], + () => {}, + context, + ); + expect(result).toEqual({ stored: 3 }); + expect((await storage.document(copyId, "current", context))?.value).toEqual({ count: 2 }); + })); + + it("refuses a batch whose copy fails before it writes anything, keeping the batches before it", () => + withStorage("batch-copy-refused", async (storage) => { + const sourceId = await storage.mintId(); + await commitWrites( + storage, + [ + { + type: "document.create", + record: sourceRecord(sourceId), + content: { kind: "base", version: 1, value: { count: 1 } }, + }, + ], + context, + ); + const first = await storage.mintId(); + const taskId = await storage.mintId>(); + const copyId = await storage.mintId(); + const result = await storage.commit( + [ + [entryWrite(first)], + // The task comes before the copy: the refusal must leave it unwritten. + [ + { type: "task", value: pendingTask(taskId) }, + { + type: "document.copy", + record: { ...sourceRecord(copyId), kind: "mismatch" }, + source: { id: sourceId, at: "current" }, + }, + ], + ], + () => {}, + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toContain(`Document copy ${copyId} was rejected`); + expect(await storage.entry(first, context)).toBeDefined(); + expect(await storage.task(taskId, context)).toBeUndefined(); + expect(await storage.document(copyId, "current", context)).toBeUndefined(); + })); + + it("rejects with nothing stored when IndexedDB fails during a batch's checks", () => + withStorage("batch-check-failure", async (storage) => { + const first = await storage.mintId(); + const second = await storage.mintId(); + const get = FakeIDBObjectStore.prototype.get; + const failure = new DOMException("disk gone", "UnknownError"); + const spy = vi.spyOn(FakeIDBObjectStore.prototype, "get").mockImplementation(function ( + this: InstanceType, + key: Parameters[0], + ) { + if (this.name === "record_ids" && key === second) throw failure; + return get.call(this, key); + }); + const receipts: number[] = []; + try { + await expect( + storage.commit([[entryWrite(first)], [entryWrite(second)]], (index) => receipts.push(index), context), + ).rejects.toBe(failure); + } finally { + spy.mockRestore(); + } + expect(receipts).toEqual([]); + expect(await storage.entry(first, context)).toBeUndefined(); + await commitWrites(storage, [entryWrite(first)], context); + expect(await storage.entry(first, context)).toBeDefined(); + })); + + it("rejects with nothing stored when IndexedDB fails during a batch's writes", () => + withStorage("batch-write-failure", async (storage) => { + const first = await storage.mintId(); + const second = await storage.mintId(); + // A function cannot be stored: the put throws IndexedDB's `DataCloneError`. + const unstorable = { + type: "entry", + value: { id: second, conversationId: ROOT_CONVERSATION_ID, kind: "batch", data: { f: () => {} } }, + } as unknown as StorageWrite; + const receipts: number[] = []; + await expect( + storage.commit([[entryWrite(first)], [unstorable]], (index) => receipts.push(index), context), + ).rejects.toMatchObject({ name: "DataCloneError" }); + expect(receipts).toEqual([]); + expect(await storage.entry(first, context)).toBeUndefined(); + expect(await storage.entry(second, context)).toBeUndefined(); + })); +}); diff --git a/packages/durable/test/jsonl-storage.test.ts b/packages/durable/test/jsonl-storage.test.ts index 8f16ae547..0a30ecf01 100644 --- a/packages/durable/test/jsonl-storage.test.ts +++ b/packages/durable/test/jsonl-storage.test.ts @@ -10,7 +10,9 @@ import { NodeExecutionEnv } from "../src/env/node.ts"; import { idFromNumber } from "../src/ids.ts"; import { JsonlStorage } from "../src/storage/jsonl/index.ts"; import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import type { + CommitResult, ConversationId, DocumentCreate, DocumentId, @@ -24,9 +26,6 @@ import type { } from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - const context = BACKGROUND_CONTEXT; type StoredTask = TaskRecord; const tempDirectories = new Set(); @@ -69,10 +68,14 @@ class ReopeningStorage implements Storage { this.directory = directory; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { - if (this.closed) return this.current.commit(writes, adopt, commitContext); + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.closed) return this.current.commit(batches, receive, commitContext); try { - return await this.current.commit(writes, adopt, commitContext); + return await this.current.commit(batches, receive, commitContext); } finally { await this.current.close(context); this.current = await JsonlStorage.open(this.directory, new NodeExecutionEnv({ cwd: this.directory }), context); @@ -163,7 +166,7 @@ function terminalTask(id: TaskId): StoredTask { } async function createRoot(storage: Storage): Promise { - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); } function sessionDocument(id: DocumentId, kind = "test.document"): DocumentCreate { @@ -332,8 +335,9 @@ describe("Pico JsonlStorage publication and recovery", () => { await createRoot(storage); const taskId = await storage.mintId>(); const documentId = await storage.mintId(); - await storage.commit([{ type: "task", value: pendingTask(taskId) }], adopt, context); - await storage.commit( + await commitWrites(storage, [{ type: "task", value: pendingTask(taskId) }], context); + await commitWrites( + storage, [ { type: "document.create", @@ -341,15 +345,15 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id: documentId, content: { kind: "delta", version: 1, ops: [] } }], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -357,14 +361,13 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 1 } }, }, ], - adopt, context, ); expect(await readLines(join(directory, `task-${taskId}.jsonl`))).toHaveLength(1); expect(await readLines(join(directory, `doc-${documentId}.jsonl`))).toHaveLength(1); - await storage.commit([{ type: "document.retire", id: documentId }], adopt, context); - await storage.commit([{ type: "task", value: terminalTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id: documentId }], context); + await commitWrites(storage, [{ type: "task", value: terminalTask(taskId) }], context); expect(await readLines(join(directory, "main.jsonl"))).toHaveLength(7); expect(await fileExists(join(directory, `task-${taskId}.jsonl`))).toBe(false); @@ -380,12 +383,12 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const taskId = await storage.mintId>(); - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId, "first") }, { type: "task", value: pendingTask(taskId, "second") }, ], - adopt, context, ); expect(await readLines(join(directory, `task-${taskId}.jsonl`))).toHaveLength(2); @@ -402,12 +405,12 @@ describe("Pico JsonlStorage publication and recovery", () => { const taskId = await storage.mintId>(); env.fail({ operation: "append", call: 1, mode: "short" }); await expect( - storage.commit( + commitWrites( + storage, [ { type: "task", value: pendingTask(taskId, "first") }, { type: "task", value: pendingTask(taskId, `second-${"x".repeat(512)}`) }, ], - adopt, context, ), ).rejects.toThrow("poisoned"); @@ -420,6 +423,46 @@ describe("Pico JsonlStorage publication and recovery", () => { expect((await stat(join(directory, `task-${taskId}.jsonl`))).size).toBe(0); }); + it("poisons the storage when an append fails after its batches applied, before a waiting read goes on (review)", async () => { + const directory = await tempDirectory(); + const gate = Promise.withResolvers(); + class FailingJoin extends NodeExecutionEnv { + fail = false; + override async joinPath(parts: string[], callContext: Context): Promise> { + if (!this.fail) return super.joinPath(parts, callContext); + await gate.promise; + return err(new FileError("unknown", "injected path resolution failure")); + } + } + const fs = new FailingJoin({ cwd: directory }); + const storage = await JsonlStorage.open(directory, fs, context); + fs.fail = true; + const conversationId = idFromNumber(100); + const committing = storage.commit( + [ + [{ type: "conversation", value: { id: conversationId } }], + [ + { + type: "document.create", + record: { id: idFromNumber(101), kind: "state", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: {} }, + }, + ], + ], + () => {}, + context, + ); + // Started while the batches are in memory and the append waits. + const read = storage.conversation(conversationId, context); + gate.resolve(); + await expect(committing).rejects.toThrow("poisoned"); + await expect(read).rejects.toThrow("poisoned"); + await storage.close(context); + const reopened = await openStorage(directory); + expect(await reopened.conversation(conversationId, context)).toBeUndefined(); + await reopened.close(context); + }); + it("serializes the complete candidate before I/O and leaves preparation failures usable", async () => { const directory = await tempDirectory(); const env = new InstrumentedEnv({ cwd: directory }); @@ -428,7 +471,8 @@ describe("Pico JsonlStorage publication and recovery", () => { env.clear(); const id = await storage.mintId(); await expect( - storage.commit( + commitWrites( + storage, [ { type: "entry", @@ -440,16 +484,15 @@ describe("Pico JsonlStorage publication and recovery", () => { }, }, ], - adopt, context, ), ).rejects.toThrow(); expect(env.operations).toEqual([]); expect(await storage.entry(id, context)).toBeUndefined(); expect( - await storage.commit( + await commitWrites( + storage, [{ type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "good" } }], - adopt, context, ), ).toBe(2); @@ -474,7 +517,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const secondId = await storage.mintId(); env.fail(failure); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.create", @@ -487,7 +531,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { text: "β" } }, }, ], - adopt, context, ), ).rejects.toThrow("poisoned"); @@ -519,7 +562,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const secondId = await storage.mintId(); env.fail(failure); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.create", @@ -532,7 +576,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ), ).rejects.toThrow("poisoned"); @@ -551,7 +594,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const firstId = await storage.mintId(); const secondId = await storage.mintId(); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -564,7 +608,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); const expected = [ @@ -576,7 +619,8 @@ describe("Pico JsonlStorage publication and recovery", () => { expect(env.operations).toEqual(expected); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -584,7 +628,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { checkpoint: true } }, }, ], - adopt, context, ); expect(env.operations).toEqual([ @@ -598,7 +641,8 @@ describe("Pico JsonlStorage publication and recovery", () => { ]); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -609,15 +653,14 @@ describe("Pico JsonlStorage publication and recovery", () => { }, }, ], - adopt, context, ); expect(env.operations).toEqual(["append:main.jsonl"]); const taskId = await storage.mintId>(); - await storage.commit([{ type: "task", value: pendingTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: pendingTask(taskId) }], context); env.clear(); - await storage.commit([{ type: "task", value: terminalTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: terminalTask(taskId) }], context); expect(env.operations).toEqual([ "append:main.jsonl", ...(fsync ? ["flush:main.jsonl"] : []), @@ -639,7 +682,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -647,10 +691,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -658,13 +702,13 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 1]] }, }, ], - adopt, context, ); env.fail(failure); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.change", @@ -672,7 +716,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).resolves.toBe(4); @@ -694,7 +737,8 @@ describe("Pico JsonlStorage publication and recovery", () => { await createRoot(storage); const taskId = await storage.mintId>(); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId) }, { @@ -703,11 +747,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 1 } }, }, ], - adopt, context, ); env.fail({ operation: "remove", call: 1, mode }); - await expect(storage.commit([{ type: "document.retire", id }], adopt, context)).resolves.toBe(3); + await expect(commitWrites(storage, [{ type: "document.retire", id }], context)).resolves.toBe(3); expect(await storage.document(id, "current", context)).toBeUndefined(); await storage.close(context); @@ -726,7 +769,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env, { fsync: true }); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -734,12 +778,12 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); env.fail({ operation: "flush", call: 2, mode }); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.change", @@ -747,7 +791,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).resolves.toBe(3); @@ -782,7 +825,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env, { fsync: true }); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -790,12 +834,12 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); env.fail({ operation: "flush", call: 3, mode }); await expect( - storage.commit( + commitWrites( + storage, [ { type: "document.change", @@ -803,12 +847,12 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ), ).resolves.toBe(3); env.clear(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -816,7 +860,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 3]] }, }, ], - adopt, context, ); await storage.close(context); @@ -834,9 +877,9 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory, env); await createRoot(storage); const id = await storage.mintId>(); - await storage.commit([{ type: "task", value: pendingTask(id) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: pendingTask(id) }], context); env.fail({ operation: "remove", call: 1, mode }); - await expect(storage.commit([{ type: "task", value: terminalTask(id) }], adopt, context)).resolves.toBe(3); + await expect(commitWrites(storage, [{ type: "task", value: terminalTask(id) }], context)).resolves.toBe(3); expect(await storage.task(id, context)).toEqual(terminalTask(id)); await storage.close(context); @@ -852,7 +895,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -860,10 +904,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -871,10 +915,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 10 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -882,7 +926,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 11]] }, }, ], - adopt, context, ); expect(await readLines(join(directory, `doc-${id}.jsonl`))).toHaveLength(2); @@ -902,12 +945,13 @@ describe("Pico JsonlStorage publication and recovery", () => { history: "rewindable" as const, fork: "asOf" as const, }; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 0 } } }], - adopt, context, ); - const changedAt = await storage.commit( + const changedAt = await commitWrites( + storage, [ { type: "document.change", @@ -915,10 +959,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 1]] }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -926,10 +970,9 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 2 } }, }, ], - adopt, context, ); - await storage.commit([{ type: "document.retire", id }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id }], context); expect(await readLines(join(directory, `doc-${id}.jsonl`))).toHaveLength(3); const reopened = await openStorage(directory); @@ -947,7 +990,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const sessionId = await storage.mintId(); const latestId = await storage.mintId(); const taskDocumentId = await storage.mintId(); - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [ { type: "task", value: pendingTask(taskId) }, { @@ -972,17 +1016,16 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); - const retiredAt = await storage.commit( + const retiredAt = await commitWrites( + storage, [ { type: "document.retire", id: sessionId }, { type: "document.retire", id: latestId }, { type: "document.retire", id: taskDocumentId }, { type: "task", value: terminalTask(taskId) }, ], - adopt, context, ); for (const file of [ @@ -1012,7 +1055,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1020,7 +1064,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { text: "kept" } }, }, ], - adopt, context, ); const sidecarPath = join(directory, `doc-${documentId}.jsonl`); @@ -1035,9 +1078,9 @@ describe("Pico JsonlStorage publication and recovery", () => { expect((await stat(sidecarPath)).size).toBe(sidecarSize); expect((await stat(mainPath)).size).toBe(mainSize); expect( - await reopened.commit( + await commitWrites( + reopened, [{ type: "document.change", id: documentId, content: { kind: "delta", version: 1, ops: [] } }], - adopt, context, ), ).toBe(3); @@ -1048,8 +1091,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const taskId = await storage.mintId>(); - await storage.commit([{ type: "task", value: pendingTask(taskId) }], adopt, context); - await storage.commit([{ type: "task", value: terminalTask(taskId) }], adopt, context); + await commitWrites(storage, [{ type: "task", value: pendingTask(taskId) }], context); + await commitWrites(storage, [{ type: "task", value: terminalTask(taskId) }], context); const sidecarPath = join(directory, `task-${taskId}.jsonl`); expect(await fileExists(sidecarPath)).toBe(false); await writeFile( @@ -1067,7 +1110,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const reopened = await openStorage(directory); expect(await fileExists(sidecarPath)).toBe(false); expect(await reopened.task(taskId, context)).toEqual(terminalTask(taskId)); - expect(await reopened.commit([], adopt, context)).toBe(4); + const next = await reopened.mintId(); + expect(await commitWrites(reopened, [{ type: "conversation", value: { id: next } }], context)).toBe(4); }); it("fails open when confirmed sidecar data is missing", async () => { @@ -1075,7 +1119,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1083,7 +1128,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); await writeFile(join(directory, `doc-${documentId}.jsonl`), ""); @@ -1097,7 +1141,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1105,10 +1150,10 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -1116,7 +1161,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], 1]] }, }, ], - adopt, context, ); const path = join(directory, `doc-${documentId}.jsonl`); @@ -1151,7 +1195,8 @@ describe("Pico JsonlStorage publication and recovery", () => { const storage = await openStorage(directory); await createRoot(storage); const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -1159,7 +1204,6 @@ describe("Pico JsonlStorage publication and recovery", () => { content: { kind: "base", version: 1, value: {} }, }, ], - adopt, context, ); const path = join(directory, `doc-${documentId}.jsonl`); @@ -1192,3 +1236,68 @@ describe("Pico JsonlStorage publication and recovery", () => { ).rejects.toThrow("Malformed complete doc-99.jsonl"); }); }); + +describe("JsonlStorage batches", () => { + it("appends a batch's markers in one write and recovers a shorter prefix from a torn one", async () => { + const directory = await tempDirectory(); + const storage = await openStorage(directory); + await createRoot(storage); + const documentId = await storage.mintId(); + await commitWrites( + storage, + [ + { + type: "document.create", + record: sessionDocument(documentId), + content: { kind: "base", version: 1, value: { n: 0 } }, + }, + ], + context, + ); + const mainPath = join(directory, "main.jsonl"); + const before = (await readFile(mainPath, "utf8")).length; + const entries = [ + await storage.mintId(), + await storage.mintId(), + await storage.mintId(), + ]; + const batches: StorageWrite[][] = entries.map((id, index) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "note" } }, + { + type: "document.change", + id: documentId, + content: { kind: "delta", version: 1, ops: [["s", ["n"], index + 1]] }, + }, + ]); + // Reads wait for the append: a read started while it runs sees every batch durable. + const result = storage.commit(batches, () => {}, context); + const read = storage.entry(entries[2]!, context); + expect(await result).toEqual({ stored: 3 }); + expect(await read).toBeDefined(); + const main = await readFile(mainPath, "utf8"); + const markers = main + .slice(before) + .split("\n") + .filter((line) => line.length > 0); + expect(markers).toHaveLength(3); + await storage.close(context); + openStorages.delete(storage); + + // Torn inside the last marker: the two complete ones survive, and the third's sidecar record is unconfirmed. + const tornAt = main.length - Math.floor(markers[2]!.length / 2); + await writeFile(mainPath, main.slice(0, tornAt)); + const reopened = await openStorage(directory); + expect(await reopened.entry(entries[0]!, context)).toBeDefined(); + expect(await reopened.entry(entries[1]!, context)).toBeDefined(); + expect(await reopened.entry(entries[2]!, context)).toBeUndefined(); + expect((await reopened.document(documentId, "current", context))?.value).toEqual({ n: 2 }); + const seq = await commitWrites( + reopened, + [{ type: "document.change", id: documentId, content: { kind: "delta", version: 1, ops: [["s", ["n"], 9]] } }], + context, + ); + expect(seq).toBe(5); + const again = await openStorage(directory); + expect((await again.document(documentId, "current", context))?.value).toEqual({ n: 9 }); + }); +}); diff --git a/packages/durable/test/memory-storage.test.ts b/packages/durable/test/memory-storage.test.ts index 6312e9db7..1b4bc9c5b 100644 --- a/packages/durable/test/memory-storage.test.ts +++ b/packages/durable/test/memory-storage.test.ts @@ -3,13 +3,14 @@ import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; import { describe, expect, it } from "vitest"; import { idFromNumber, seqFromNumber } from "../src/ids.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; -import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import { type EntryId, ROOT_CONVERSATION_ID, type Seq, type StorageWrite } from "../src/types.ts"; registerStorageConformance({ describe, expect, it }, "MemoryStorage", (use) => use(new MemoryStorage())); it("does not expose retained state through a prepared commit", async () => { const storage = new MemoryStorage(); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], () => {}, BACKGROUND_CONTEXT); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); const entryId = idFromNumber(2); const prepared = storage.prepareCommit([ { @@ -30,3 +31,41 @@ it("does not expose retained state through a prepared commit", async () => { expect(prepared.apply()).toBe(seqFromNumber(2)); expect((await storage.entry(entryId, BACKGROUND_CONTEXT))?.entry.data).toEqual({ nested: [1] }); }); + +it("delivers the receipts of the batches applied before one whose apply throws, then rejects", async () => { + const failure = new Error("apply failed"); + class FailingApply extends MemoryStorage { + failAt: Seq | undefined; + override prepareCommit(writes: readonly StorageWrite[], seq?: Seq) { + const prepared = super.prepareCommit(writes, seq); + if (prepared.seq !== this.failAt) return prepared; + return { + ...prepared, + apply: (): Seq => { + throw failure; + }, + }; + } + } + const storage = new FailingApply(); + const root = await commitWrites( + storage, + [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], + BACKGROUND_CONTEXT, + ); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + storage.failAt = seqFromNumber(root + 2); + const receipts: [number, Seq][] = []; + await expect( + storage.commit( + [first, second].map((id) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "m" } }, + ]), + (index, seq) => receipts.push([index, seq]), + BACKGROUND_CONTEXT, + ), + ).rejects.toBe(failure); + expect(receipts).toEqual([[0, root + 1]]); + expect((await storage.entry(first, BACKGROUND_CONTEXT))?.commitSeq).toBe(root + 1); + expect(await storage.entry(second, BACKGROUND_CONTEXT)).toBeUndefined(); +}); diff --git a/packages/durable/test/nested-images-bench.ts b/packages/durable/test/nested-images-bench.ts index f8c0e6d19..858f21672 100644 --- a/packages/durable/test/nested-images-bench.ts +++ b/packages/durable/test/nested-images-bench.ts @@ -10,6 +10,7 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/nested-images-bench.ts [count] */ + import { randomBytes } from "node:crypto"; import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -18,7 +19,7 @@ import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import type { ImageContent } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { arr, lit, num, obj, opt, str } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, @@ -95,11 +96,11 @@ function noWrites(): Written { }; } -const ImageArgument = Type.Object({ - type: Type.Literal("image"), - mimeType: Type.String(), - data: Type.Optional(Type.String()), - source: Type.Optional(Type.Object({ taskId: Type.Number() })), +const ImageArgument = obj({ + type: lit("image"), + mimeType: str(), + data: opt(str()), + source: opt(obj({ taskId: num() })), }); function registryFor(scenario: Scenario, probe: Probe) { @@ -107,7 +108,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const readImage = defineTool({ name: "read_image", description: "Read an image", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), // One image, so programs get it as an `ImageContent`. Unique per call: base64 of random bytes. execute: async () => ({ output: [ @@ -118,7 +119,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const classify = defineTool({ name: "classify", description: "Classify images", - parameters: Type.Object({ images: Type.Array(ImageArgument) }), + parameters: obj({ images: arr(ImageArgument) }), execute: async (args, api, callContext) => { for (const image of args.images) { let data = image.data; @@ -141,7 +142,7 @@ function registryFor(scenario: Scenario, probe: Probe) { const script = defineTool({ name: "script", description: "Read and classify images", - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), callers: ["model"], execute: async (args, api, callContext) => { for (let n = 0; n < args.count; n++) { @@ -189,12 +190,12 @@ async function runScenario(scenario: Scenario): Promise { let commits = 0; const written = noWrites(); const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; + storage.commit = (batches, receive, commitContext) => { + commits += batches.length; // SQLite commits synchronously, which can starve the timer; sample here too. sample(); - for (const write of writes) written[write.type] += JSON.stringify(write).length; - return commit(writes, adopt, commitContext); + for (const write of batches.flat()) written[write.type] += JSON.stringify(write).length; + return commit(batches, receive, commitContext); }; const harness = await Harness.open(storage, { models, registry }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); diff --git a/packages/durable/test/nested-tools-bench.ts b/packages/durable/test/nested-tools-bench.ts index 57561608c..002383fec 100644 --- a/packages/durable/test/nested-tools-bench.ts +++ b/packages/durable/test/nested-tools-bench.ts @@ -9,13 +9,14 @@ * * node --conditions=source --experimental-strip-types --expose-gc test/nested-tools-bench.ts [streaming|nesting] */ + import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { type Conversation, createRegistry, @@ -103,7 +104,7 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB const blob = defineTool({ name: "blob", description: "Stream a blob", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async (_args, api) => { for (let n = 0; n < CHUNKS; n++) { api.output(chunk); @@ -118,7 +119,7 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB defineTool({ name, description: `Call ${target} in parallel`, - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), execute: async (args, api, callContext) => { const calls = Array.from({ length: args.count }, (_, n) => api.executeTool(target, { n, count: FAN_OUT }, callContext, options), @@ -133,10 +134,10 @@ async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedB let commits = 0; const written = noWrites(); const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; - for (const write of writes) written[write.type] += JSON.stringify(write).length; - return commit(writes, adopt, commitContext); + storage.commit = (batches, receive, commitContext) => { + commits += batches.length; + for (const write of batches.flat()) written[write.type] += JSON.stringify(write).length; + return commit(batches, receive, commitContext); }; const harness = await Harness.open(storage, { models, registry }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); diff --git a/packages/durable/test/scheduler-fanout-bench.ts b/packages/durable/test/scheduler-fanout-bench.ts index ed3bb2eaa..2d2fc381f 100644 --- a/packages/durable/test/scheduler-fanout-bench.ts +++ b/packages/durable/test/scheduler-fanout-bench.ts @@ -5,10 +5,11 @@ * * node --conditions=source --experimental-strip-types test/scheduler-fanout-bench.ts [calls...] */ + import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { num, obj } from "@earendil-works/pi-schema"; import { createRegistry, defineExtension, defineTool, Harness, ToolResultEntry } from "../src/index.ts"; import { MemoryStorage } from "../src/storage/memory.ts"; @@ -20,13 +21,13 @@ async function run(calls: number): Promise<{ ms: number; commits: number }> { const leaf = defineTool({ name: "leaf", description: "Return at once", - parameters: Type.Object({ n: Type.Number() }), + parameters: obj({ n: num() }), execute: async () => ({ output: [{ type: "text", text: "ok" }] }), }); const fan = defineTool({ name: "fan", description: "Call leaf in parallel", - parameters: Type.Object({ count: Type.Number() }), + parameters: obj({ count: num() }), execute: async (args, api, callContext) => { const results = await Promise.all( Array.from({ length: args.count }, (_, n) => @@ -41,9 +42,9 @@ async function run(calls: number): Promise<{ ms: number; commits: number }> { const storage = new MemoryStorage(); let commits = 0; const commit = storage.commit.bind(storage); - storage.commit = (writes, adopt, commitContext) => { - commits++; - return commit(writes, adopt, commitContext); + storage.commit = (batches, receive, commitContext) => { + commits += batches.length; + return commit(batches, receive, commitContext); }; const harness = await Harness.open(storage, { models, registry }, context); const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); diff --git a/packages/durable/test/search.test.ts b/packages/durable/test/search.test.ts new file mode 100644 index 000000000..c4510cd46 --- /dev/null +++ b/packages/durable/test/search.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import { rankDocuments, type SearchDocument, tokenize } from "../src/search.ts"; + +function doc(name: string, text: string, namespaceText?: string): SearchDocument { + return namespaceText === undefined ? { name, text } : { name, text, namespaceText }; +} + +describe("tokenize", () => { + it("splits camelCase and snake_case, drops stop words, and folds plurals", () => { + expect(tokenize("listIssues for the GitHub_repo")).toEqual(["list", "issue", "git", "hub", "repo"]); + expect(tokenize("searches queries HTTPServer")).toEqual(["search", "query", "http", "server"]); + expect(tokenize("repo.issues/search:queries")).toEqual(["repo", "issue", "search", "query"]); + }); + + it.each([ + ["搜索仓库问题", ["搜索", "仓库", "问题"]], + ["文書を検索", ["文書", "を", "検索"]], + ["문서 검색", ["문서", "검색"]], + ["ค้นหาเอกสาร", ["ค้นหา", "เอกสาร"]], + ["Поиск документов", ["поиск", "документов"]], + ["البحث عن الملفات", ["البحث", "عن", "الملفات"]], + ["फ़ाइल खोजें", ["फ़ाइल", "खोजें"]], + ["Éléments cafés", ["éléments", "cafés"]], + ])("segments Unicode words in %s", (text, expected) => { + expect(tokenize(text)).toEqual(expected); + }); + + it("preserves non-ASCII words and splits mixed-language identifiers", () => { + expect(tokenize("搜索GitHub_repoIssues résumé cafés")).toEqual([ + "搜索", + "git", + "hub", + "repo", + "issue", + "résumé", + "cafés", + ]); + }); + + it("normalizes canonically equivalent Unicode text", () => { + expect(tokenize("CAFÉ cafe\u0301")).toEqual(["café", "café"]); + }); + + it("ignores whitespace, punctuation, and emoji", () => { + expect(tokenize(" \t，。!? 😀 ")).toEqual([]); + }); +}); + +describe("rankDocuments", () => { + const documents = [ + doc("mcp__github__list_issues", "List issues in a repository. state open or closed"), + doc("mcp__github__create_pull_request", "Open a pull request."), + doc("mcp__linear__search_issues", "Search Linear issues by text."), + doc("mcp__docs__search", "Search the documentation."), + ]; + + it("ranks by term relevance and respects the limit", () => { + expect(rankDocuments("issue", documents, 8)).toEqual(["mcp__linear__search_issues", "mcp__github__list_issues"]); + expect(rankDocuments("pull requests", documents, 8)[0]).toBe("mcp__github__create_pull_request"); + expect(rankDocuments("search", documents, 1)).toHaveLength(1); + expect(rankDocuments("closed", documents, 8)).toEqual(["mcp__github__list_issues"]); + }); + + it.each([ + ["搜索仓库问题", "问题"], + ["文書を検索", "検索"], + ["문서 검색", "검색"], + ["ค้นหาเอกสาร", "เอกสาร"], + ["Поиск документов", "документов"], + ["البحث عن الملفات", "الملفات"], + ["फ़ाइल खोजें", "खोजें"], + ["Rechercher un café", "cafe\u0301"], + ])("matches Unicode queries against %s", (text, query) => { + expect(rankDocuments(query, [...documents, doc("localized", text)], 8)).toEqual(["localized"]); + }); + + it("returns nothing for unknown or empty queries", () => { + expect(rankDocuments("kubernetes", documents, 8)).toEqual([]); + expect(rankDocuments("the", documents, 8)).toEqual([]); + expect(rankDocuments("未知", documents, 8)).toEqual([]); + }); + + it("searches the namespace text", () => { + const namespaced = [doc("mcp__x__run", "Run it.", "mcp__x Kubernetes cluster tools")]; + expect(rankDocuments("kubernetes", namespaced, 8)).toEqual(["mcp__x__run"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query is its name", () => { + const named = [ + doc("mcp__app__queue_read", "Read a queued message of this thread."), + doc( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans. Continue with afterPosition until hasMore is false.", + ), + ]; + for (const query of ["thread_read", "dev_app__thread_read", "mcp__dev-app__thread_read", "Thread_Read"]) { + expect(rankDocuments(query, named, 1)).toEqual(["mcp__dev-app__thread_read"]); + } + // Partial names still rank by BM25. + expect(rankDocuments("read", named, 1)).toEqual(["mcp__app__queue_read"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a name match above mentions in descriptions", () => { + const ci = [ + doc("mcp__ci__status", "Deploy status: last deploy, deploy log, deploy errors."), + doc( + "mcp__ci__deploy_release", + "Start a release of the current branch to an environment, run the migrations, switch traffic over, and wait until the new version is live and healthy.", + ), + ]; + expect(rankDocuments("deploy", ci, 1)).toEqual(["mcp__ci__deploy_release"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool first when the query contains its whole name", () => { + const peekaboo = "mcp__peekaboo Peekaboo controls macOS apps by app name."; + const named = [ + doc("mcp__peekaboo__see", "Capture the app window of the peekaboo app and describe the app.", peekaboo), + doc("mcp__peekaboo__menu", "Click a menu item of an app.", peekaboo), + doc( + "mcp__peekaboo__app", + "Launch, quit, relaunch, focus, hide, or switch applications. Targets an application by name, bundle identifier, or process ID, and can wait until it is ready.", + peekaboo, + ), + doc("mcp__app__queue_read", "Read a queued message of this thread."), + doc( + "mcp__dev-app__thread_read", + "Read durable state and a paginated timeline from any thread. The default view returns user messages, assistant messages and plans.", + ), + ]; + expect(rankDocuments("peekaboo app", named, 1)).toEqual(["mcp__peekaboo__app"]); + expect(rankDocuments("open the peekaboo app tool", named, 1)).toEqual(["mcp__peekaboo__app"]); + expect(rankDocuments("read thread_read state", named, 1)).toEqual(["mcp__dev-app__thread_read"]); + // Without the whole name, mentions in descriptions decide. + expect(rankDocuments("describe window", named, 1)).toEqual(["mcp__peekaboo__see"]); + }); + + // https://github.com/earendil-works/pi/issues/11331 + it("ranks a tool matching more query terms first, whatever the scores", () => { + const peekaboo = [ + doc("mcp__peekaboo__app", "Launch, quit, focus, or switch applications by name or bundle identifier."), + doc("mcp__peekaboo__see", "Capture the app window and describe the UI elements of the app."), + doc("mcp__peekaboo__window", "Move, resize, or close a window of an app."), + doc("mcp__peekaboo__image", "Take a screenshot of the screen, an app, or a window."), + ]; + // `window` and `app` are whole names, but each covers one of the two terms. + expect(rankDocuments("screenshot window", peekaboo, 1)).toEqual(["mcp__peekaboo__image"]); + expect(rankDocuments("describe app", peekaboo, 1)).toEqual(["mcp__peekaboo__see"]); + // An exact name still ranks first. + expect(rankDocuments("window", peekaboo, 1)).toEqual(["mcp__peekaboo__window"]); + }); +}); diff --git a/packages/durable/test/session-adoption.test.ts b/packages/durable/test/session-adoption.test.ts index ddd7dd1fd..a83da3181 100644 --- a/packages/durable/test/session-adoption.test.ts +++ b/packages/durable/test/session-adoption.test.ts @@ -4,6 +4,7 @@ import { basename, join } from "node:path"; import type { Context } from "@earendil-works/chord"; import { type CommitPublication, + type CommitResult, defineDoc, MemoryStorage, type Seq, @@ -41,22 +42,26 @@ class FaultyStorage extends ControlledStorage { fault: "reject_after_adopt" | "skip_adopt" | undefined; readonly order: string[] = []; - override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, callContext: Context) { + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + callContext: Context, + ): Promise { const fault = this.fault; this.fault = undefined; - if (fault === "skip_adopt") return super.commit(writes, () => {}, callContext); - const seq = await super.commit( - writes, - (committed) => { + if (fault === "skip_adopt") return super.commit(batches, () => {}, callContext); + const result = await super.commit( + batches, + (index, committed) => { this.order.push("adopt"); - adopt(committed); + receive(index, committed); this.order.push("adopted"); }, callContext, ); this.order.push("resolve"); if (fault === "reject_after_adopt") throw new Error("reclaim failed"); - return seq; + return result; } } @@ -75,8 +80,8 @@ afterEach(() => { vi.restoreAllMocks(); }); -describe("Session adoption inside Storage.commit()", () => { - it("adopts synchronously inside the commit, before Storage resolves", async () => { +describe("Session adoption after Storage stored a commit", () => { + it("takes the durability receipt inside the commit, and adopts once Storage resolved", async () => { const { storage, session } = open(); const adopt = vi.spyOn(Transaction.prototype, "adopt"); await bump(session); @@ -128,8 +133,8 @@ describe("Session adoption inside Storage.commit()", () => { }); }); -describe("publication at adoption", () => { - it("runs its own observers inside adopt and host listeners one microtask later, outside it, before Storage resolves", async () => { +describe("publication after storage", () => { + it("runs its own observers once Storage resolved, and host listeners one microtask later", async () => { const { storage, session } = open(); session.followCommits(() => { storage.order.push(`observed adopting=${storage.adopting}`); @@ -141,12 +146,12 @@ describe("publication at adoption", () => { end: () => {}, }); await bump(session); - expect(storage.order).toEqual(["adopt", "observed adopting=true", "adopted", "host adopting=false", "resolve"]); + expect(storage.order).toEqual(["adopt", "adopted", "resolve", "observed adopting=false", "host adopting=false"]); await session.close(context); }); - it("calls the host listeners registered when its dispatch runs, and none for a commit adopted without one", async () => { - const { storage, session } = open(); + it("calls the host listeners registered when its dispatch runs, and none for a commit published without one", async () => { + const { session } = open(); const calls: string[] = []; const removed = session.observeCommits({ advance: () => { @@ -160,7 +165,11 @@ describe("publication at adoption", () => { }, end: () => {}, }); - storage.afterAdopt = () => { + // During the publication, after the dispatch was queued. + let changed = false; + session.followCommits(() => { + if (changed) return; + changed = true; removed.detach(); session.observeCommits({ advance: () => { @@ -168,20 +177,23 @@ describe("publication at adoption", () => { }, end: () => {}, }); - }; + }); await bump(session); expect(calls).toEqual(["kept", "added"]); - // A Session with no host listener at adoption queues no dispatch, so one added before Storage resolves misses it. + // A Session with no host listener at the publication queues no dispatch, so one added during it misses it. const bare = open(); const late: Seq[] = []; - bare.storage.afterAdopt = () => { + let added = false; + bare.session.followCommits(() => { + if (added) return; + added = true; bare.session.observeCommits({ advance: (publication) => { late.push(publication.seq); }, end: () => {}, }); - }; + }); await bump(bare.session); expect(late).toEqual([]); await bump(bare.session); @@ -210,14 +222,14 @@ describe("publication at adoption", () => { storage.order.push("close"); }); await expect(bump(session)).resolves.toBeUndefined(); - // `adopted` and `resolve`: adopt returned to Storage without throwing, and the Session failed only after that. + // Published once Storage resolved; the Session failed only after the hosts got the commit. expect(storage.order).toEqual([ "adopt", + "adopted", + "resolve", "observed", "observed after", - "adopted", "host failed=false", - "resolve", "close", ]); expect(await session.closed).toEqual({ kind: "failed", error: bug }); @@ -243,7 +255,7 @@ describe("publication at adoption", () => { storage.order.length = 0; storage.fault = "reject_after_adopt"; await expect(bump(session)).rejects.toThrow("reclaim failed"); - expect(storage.order).toEqual(["adopt", "observed", "adopted", "host", "resolve"]); + expect(storage.order).toEqual(["adopt", "adopted", "resolve", "observed", "host"]); expect(stored).toHaveLength(1); expect(await session.closed).toMatchObject({ kind: "failed" }); const reopened = new SessionImpl(storage.reopen()); @@ -283,7 +295,7 @@ describe("publication at adoption", () => { await session.commit(async (tx) => { (await tx.doc(NoteDoc, conversationId)).b = 2; }, context); - expect(storage.order).toEqual(["observed", "adopt", "observed", "adopted", "host", "resolve"]); + expect(storage.order).toEqual(["observed", "adopt", "adopted", "resolve", "observed", "host"]); expect(documentValues(observed[1]!).map((change) => change.ops)).toEqual([[["s", ["b"], 2]]]); expect(stored).toHaveLength(1); expect(documentValues(stored[0]!).map((change) => change.ops)).toEqual([ @@ -296,7 +308,7 @@ describe("publication at adoption", () => { await session.close(context); }); - it("publishes a store of held changes to host listeners one microtask after its adopt", async () => { + it("publishes a store of held changes to host listeners once Storage resolved", async () => { const { storage, session } = open(); const conversationId = await createConversation(session); await session.commit(async (tx) => { @@ -317,11 +329,11 @@ describe("publication at adoption", () => { ); storage.order.length = 0; await session.unloadDocuments(); - expect(storage.order).toEqual(["adopt", "adopted", "host adopting=false", "resolve"]); + expect(storage.order).toEqual(["adopt", "adopted", "resolve", "host adopting=false"]); await session.close(context); }); - it("publishes a JSONL commit before the storage reclaims its sidecar and resolves", async () => { + it("publishes a JSONL commit once the storage reclaimed its sidecar and resolved", async () => { /** Node environment that records each sidecar rename, the last step of a reclaim. */ class RenameLog extends NodeExecutionEnv { readonly order: string[] = []; @@ -360,7 +372,7 @@ describe("publication at adoption", () => { }); await count(); env.order.push("resolved"); - expect(env.order).toEqual(["host", expect.stringMatching(/^rename doc-\d+\.jsonl$/), "resolved"]); + expect(env.order).toEqual([expect.stringMatching(/^rename doc-\d+\.jsonl$/), "host", "resolved"]); } finally { await session.close(context); await rm(directory, { recursive: true, force: true }); diff --git a/packages/durable/test/session-cached-storage.test.ts b/packages/durable/test/session-cached-storage.test.ts new file mode 100644 index 000000000..1abbb44f2 --- /dev/null +++ b/packages/durable/test/session-cached-storage.test.ts @@ -0,0 +1,183 @@ +import type { JsonValue } from "@earendil-works/chord"; +import { describe, expect, it } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { CachedStorage } from "../src/session/cached-storage.ts"; +import { MemoryStorage } from "../src/storage/memory.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import type { ConversationId, StorageWrite, TaskId, TaskRecord } from "../src/types.ts"; +import { context } from "./session-support.ts"; + +function task(id: number, status: "pending" | "running" = "pending"): TaskRecord { + return { + id: idFromNumber>(id), + conversationId: idFromNumber(1), + kind: "test.task", + version: 1, + input: null, + background: false, + abortRequested: false, + state: { status, checkpoint: { phase: "run" } }, + }; +} + +class CountingStorage extends MemoryStorage { + taskReads = 0; + override task(...args: Parameters) { + this.taskReads++; + return super.task(...args); + } +} + +/** Stores `writes` as one batch, then publishes it, as the Session does. */ +async function commit(storage: CachedStorage, writes: StorageWrite[]): Promise { + await commitWrites(storage, writes, context); + storage.published(writes); +} + +describe("cached storage", () => { + it("serves a written record without a read and follows later writes", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [{ type: "conversation", value: { id: idFromNumber(1) } }]); + await commit(storage, [{ type: "task", value: task(2) }]); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("pending"); + await commit(storage, [{ type: "task", value: task(2, "running") }]); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + expect(backend.taskReads).toBe(0); + }); + + it("keeps a read only when no commit overlapped it", async () => { + const backend = new CountingStorage(); + await commitWrites( + backend, + [ + { type: "conversation", value: { id: idFromNumber(1) } }, + { type: "task", value: task(2) }, + ], + context, + ); + const storage = new CachedStorage(backend); + const overlapped = storage.task(idFromNumber(2), context); + const written = commit(storage, [{ type: "task", value: task(2, "running") }]); + await Promise.all([overlapped, written]); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + await storage.task(idFromNumber(2), context); + expect(backend.taskReads).toBe(1); + }); + + it("keeps a bounded number of ended tasks, and every live one until it ends", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [{ type: "conversation", value: { id: idFromNumber(1) } }]); + for (let id = 2; id < 600; id++) await commit(storage, [{ type: "task", value: task(id) }]); + // More live tasks than the ended table holds: each one's next change finds it without a read. + expect(storage.sizes).toMatchObject({ tasks: 0, liveTasks: 598 }); + for (let id = 2; id < 600; id++) await storage.task(idFromNumber(id), context); + expect(backend.taskReads).toBe(0); + for (let id = 2; id < 600; id++) { + const ended = { ...task(id), state: { status: "terminal", outcome: { status: "completed", result: null } } }; + await commit(storage, [{ type: "task", value: ended as TaskRecord }]); + } + expect(storage.sizes).toMatchObject({ tasks: 256, liveTasks: 0 }); + }); + + it("applies a batch stored with commit when it is published, not at its receipt", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [ + { type: "conversation", value: { id: idFromNumber(1) } }, + { type: "task", value: task(2) }, + ]); + const batch: StorageWrite[] = [{ type: "task", value: task(2, "running") }]; + const receipts: number[] = []; + expect(await storage.commit([batch], (index) => receipts.push(index), context)).toEqual({ stored: 1 }); + expect(receipts).toEqual([0]); + // Stored, not yet published: the cache still shows what was published. + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("pending"); + storage.published(batch); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + expect(backend.taskReads).toBe(0); + }); + + it("drops a read a publication of its record overlapped, and keeps versions only for reads in flight", async () => { + const backend = new CountingStorage(); + await commitWrites( + backend, + [ + { type: "conversation", value: { id: idFromNumber(1) } }, + { type: "task", value: task(2) }, + { type: "task", value: task(3) }, + ], + context, + ); + const storage = new CachedStorage(backend); + const overlapped = storage.task(idFromNumber(2), context); + const untouched = storage.task(idFromNumber(3), context); + expect(storage.readsInFlight).toBe(2); + // A publication of another record does not drop the read of task 3. + storage.published([{ type: "task", value: task(2, "running") }]); + await Promise.all([overlapped, untouched]); + expect(storage.readsInFlight).toBe(0); + expect((await storage.task(idFromNumber(2), context))?.state.status).toBe("running"); + expect((await storage.task(idFromNumber(3), context))?.state.status).toBe("pending"); + expect(backend.taskReads).toBe(2); + }); + + it("keeps the ended tasks a task waits on until it moved on from its wait", async () => { + const backend = new CountingStorage(); + const storage = new CachedStorage(backend); + await commit(storage, [{ type: "conversation", value: { id: idFromNumber(1) } }]); + const ended = (id: number) => + ({ + ...task(id), + state: { status: "terminal", outcome: { status: "completed", result: id } }, + }) as TaskRecord; + const waiting = (state: TaskRecord["state"]) => + ({ ...task(2), state }) as TaskRecord; + const on = [3, 4].map((id) => idFromNumber(id)); + await commit(storage, [ + { + type: "task", + value: waiting({ status: "waiting", checkpoint: { phase: "read" }, on, policy: "allSettled" }), + }, + ]); + await commit(storage, [ + { type: "task", value: ended(3) }, + { type: "task", value: ended(4) }, + ]); + // A burst of other ended tasks fills the bounded table. + for (let id = 10; id < 300; id++) await commit(storage, [{ type: "task", value: ended(id) }]); + expect(storage.sizes).toMatchObject({ awaited: 2, waits: 1 }); + // Resumed: still held for the phase that reads the outcomes. + await commit(storage, [{ type: "task", value: waiting({ status: "running", checkpoint: { phase: "read" } }) }]); + for (const id of on) expect((await storage.task(id, context))?.state.status).toBe("terminal"); + expect(backend.taskReads).toBe(0); + expect(storage.sizes).toMatchObject({ awaited: 2, waits: 1 }); + // Its next change ends the hold: the records go back to the bounded table, and no wait is left. + await commit(storage, [{ type: "task", value: waiting({ status: "running", checkpoint: { phase: "next" } }) }]); + expect(storage.sizes).toMatchObject({ awaited: 0, waits: 0, tasks: 256 }); + // A wait that ends with its task leaves nothing either. + await commit(storage, [ + { + type: "task", + value: waiting({ status: "waiting", checkpoint: { phase: "read" }, on, policy: "allSettled" }), + }, + ]); + expect(storage.sizes).toMatchObject({ awaited: 2, waits: 1 }); + await commit(storage, [{ type: "task", value: ended(2) }]); + expect(storage.sizes).toMatchObject({ awaited: 0, waits: 0 }); + }); + + it("keeps the written record itself, frozen: no copy", async () => { + const storage = new CachedStorage(new MemoryStorage()); + const conversation = { id: idFromNumber(1) }; + const record = task(2); + storage.published([ + { type: "conversation", value: conversation }, + { type: "task", value: record }, + ]); + expect(await storage.task(idFromNumber(2), context)).toBe(record); + expect(await storage.conversation(idFromNumber(1), context)).toBe(conversation); + expect(Object.isFrozen(record.state)).toBe(true); + }); +}); diff --git a/packages/durable/test/session-conversation-tails.test.ts b/packages/durable/test/session-conversation-tails.test.ts new file mode 100644 index 000000000..9ed7e40e6 --- /dev/null +++ b/packages/durable/test/session-conversation-tails.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { ConversationTails, type HeadMarker } from "../src/session/conversation-tails.ts"; +import type { ConversationId, EntryId, EntryRecord, StorageWrite } from "../src/types.ts"; + +const conversation = (id: number) => idFromNumber(id); +const entryId = (id: number) => idFromNumber(id); + +function entry(id: number, conversationId: number, head?: number): StorageWrite { + const value: EntryRecord = { + id: entryId(id), + conversationId: conversation(conversationId), + kind: "note", + ...(head === undefined ? {} : { head: entryId(head) }), + }; + return { type: "entry", value }; +} + +const never = (): Promise => Promise.reject(new Error("read")); + +describe("conversation tails", () => { + it("reads once, then follows the commits", async () => { + const tails = new ConversationTails(); + let reads = 0; + const read = async () => { + reads++; + return entryId(5); + }; + expect(await tails.latest(conversation(1), read)).toBe(5); + expect(await tails.latest(conversation(1), read)).toBe(5); + expect(reads).toBe(1); + tails.apply([entry(9, 1)]); + expect(await tails.latest(conversation(1), never)).toBe(9); + }); + + it("knows a new root conversation is empty, and a fork only once read or written", async () => { + const tails = new ConversationTails(); + tails.apply([ + { type: "conversation", value: { id: conversation(2) } }, + { + type: "conversation", + value: { id: conversation(3), parent: { conversationId: conversation(2), at: entryId(4) } }, + }, + ]); + expect(await tails.latest(conversation(2), never)).toBeUndefined(); + expect(await tails.marker(conversation(2), undefined, never)).toBeUndefined(); + expect(await tails.latest(conversation(3), async () => entryId(4))).toBe(4); + }); + + it("answers a cutoff from the newest marker only at or after it", async () => { + const tails = new ConversationTails(); + tails.apply([{ type: "conversation", value: { id: conversation(1) } }, entry(10, 1), entry(11, 1, 10)]); + const marker = (await tails.marker(conversation(1), undefined, never)) as HeadMarker; + expect(marker.id).toBe(11); + expect((await tails.marker(conversation(1), entryId(12), never))?.id).toBe(11); + // An older cutoff may have an older marker: read, and not kept. + let reads = 0; + const older = async () => { + reads++; + return undefined; + }; + expect(await tails.marker(conversation(1), entryId(10), older)).toBeUndefined(); + expect(await tails.marker(conversation(1), entryId(10), older)).toBeUndefined(); + expect(reads).toBe(2); + expect((await tails.marker(conversation(1), undefined, never))?.id).toBe(11); + }); + + it("keeps a bounded number of conversations, fewer with markers", () => { + const tails = new ConversationTails(); + for (let id = 1; id <= 5000; id++) tails.apply([entry(2 * id, id), entry(2 * id + 1, id, 2 * id)]); + expect(tails.size).toBe(4096); + expect(tails.markers).toBe(256); + }); + + it("freezes the markers it keeps", () => { + const tails = new ConversationTails(); + const write = entry(2, 1, 1); + tails.apply([write]); + expect(Object.isFrozen((write as { value: object }).value)).toBe(true); + }); +}); diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index 639d695b8..8d1b5e927 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -1,7 +1,7 @@ // The Session's document cache is bounded by use: unattached documents leave it after the retention period and load // from Storage again; documents with a watch or a document state never leave it. -import type { Context } from "@earendil-works/chord"; -import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; + +import { fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { type ConversationId, defineDoc, @@ -10,9 +10,9 @@ import { type HarnessSettings, type LiveState, MemoryStorage, - type Seq, type StorageWrite, } from "@earendil-works/pi-durable"; +import { obj } from "@earendil-works/pi-schema"; import { describe, expect, it, onTestFinished } from "vitest"; import { SessionImpl, setDefaultDocumentRetention } from "../src/session/session.ts"; import { chatSetup, openChat } from "./chat-support.ts"; @@ -218,9 +218,9 @@ describe("document cache retention", () => { const conversationId = await createConversation(session); const writes: StorageWrite[][] = []; const commit = storage.commit.bind(storage); - storage.commit = (batch: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) => { - writes.push([...batch]); - return commit(batch, adopt, commitContext); + storage.commit = (batches, receive, commitContext) => { + for (const batch of batches) writes.push([...batch]); + return commit(batches, receive, commitContext); }; for (let n = 0; n < 5; n++) { await bump(session, conversationId); @@ -521,7 +521,7 @@ describe("document cache retention", () => { defineTool({ name: "steps", description: "Streams three steps", - parameters: Type.Object({}), + parameters: obj({}), execute: async (_args, api) => { for (const step of ["a\n", "b\n", "c\n"]) { api.output(step); diff --git a/packages/durable/test/session-group-commit.test.ts b/packages/durable/test/session-group-commit.test.ts new file mode 100644 index 000000000..d135c4740 --- /dev/null +++ b/packages/durable/test/session-group-commit.test.ts @@ -0,0 +1,1017 @@ +// Group commit on the Session line (docs/group-commit.md): batching, gates, outcome fences, publication, failures per +// part, upkeep through the writer, backpressure, bounds, and a randomized interleaving against an oracle. +import type { Context } from "@earendil-works/chord"; +import { + type CommitOptions, + type CommitResult, + type ConversationId, + defineDoc, + type EntryId, + type Seq, + SessionFailed, + type StorageWrite, + type TaskId, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { CachedStorage } from "../src/session/cached-storage.ts"; +import type { ObservedCommit } from "../src/session/session.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { Transaction } from "../src/session/transaction.ts"; +import { ControlledStorage, context, flush, refusingAt } from "./session-support.ts"; +import { type Deferred, deferred, eventually, settled } from "./task-support.ts"; + +/** A latest, initial-fork conversation document: one key per conversation, and one a deferred commit may hold. */ +const Note = defineDoc<{ n: number }>({ + checkpointWhen: () => false, + kind: "gc.note", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ n: 0 }), +}); + +const Shared = defineDoc<{ n: number }>({ + checkpointWhen: () => false, + kind: "gc.shared", + version: 1, + scope: "session", + initial: () => ({ n: 0 }), +}); + +const DEFERRED = { persistence: { kind: "deferred", withinMs: Number.POSITIVE_INFINITY } } as const; + +/** + * Memory storage whose `commit()` calls are recorded and can be held in flight, and whose batches can be refused + * (failed before adoption) or whose calls can reject after their receipts. + */ +class BatchStorage extends ControlledStorage { + readonly calls: (readonly (readonly StorageWrite[])[])[] = []; + /** Refuse the next batch this matches, with `refusal.value`. */ + refuse: ((writes: readonly StorageWrite[]) => boolean) | undefined; + refusal: { readonly value: unknown } = { value: new Error("refused") }; + /** Reject the call with this index after its batches are stored, with `rejection.value`. */ + rejectCall: number | undefined; + rejection: { readonly value: unknown } = { value: new Error("connection lost") }; + /** Delay each call by up to this many milliseconds, at random. */ + latency: (() => number) | undefined; + #held: { readonly entered: Deferred; readonly gate: Deferred } | undefined; + + /** Hold the next call in flight until released. */ + hold(): { readonly entered: Promise; release(): void } { + const held = { entered: deferred(), gate: deferred() }; + this.#held = held; + return { entered: held.entered.promise, release: () => held.gate.resolve() }; + } + + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + const call = this.calls.length; + this.calls.push(batches); + const held = this.#held; + this.#held = undefined; + if (held !== undefined) { + held.entered.resolve(); + await held.gate.promise; + } + const delay = this.latency?.(); + if (delay !== undefined) await new Promise((resolve) => setTimeout(resolve, delay)); + const refuses = (writes: readonly StorageWrite[]): boolean => { + if (this.refuse?.(writes) !== true) return false; + this.refuse = undefined; + return true; + }; + const result = await refusingAt( + (...args) => super.commit(...args), + batches, + receive, + commitContext, + refuses, + this.refusal.value, + ); + if (this.rejectCall === call) throw this.rejection.value; + return result; + } +} + +async function open(conversations: number): Promise<{ + readonly storage: BatchStorage; + readonly session: SessionImpl; + readonly ids: ConversationId[]; + readonly published: ObservedCommit[]; +}> { + const storage = new BatchStorage(); + const session = new SessionImpl(storage, Date.now, () => 600_000); + const ids = await session.commit(async (tx) => { + const created: ConversationId[] = []; + for (let n = 0; n < conversations; n++) { + created.push((await tx.createConversation({ ownership: { kind: "ownerless" } })).id); + } + return created; + }, context); + const published: ObservedCommit[] = []; + session.followCommits((publication) => { + published.push(publication); + }); + return { storage, session, ids, published }; +} + +/** Increment conversation `id`'s note; resolves with the value it read. */ +function bump(session: SessionImpl, id: ConversationId, options?: CommitOptions): Promise { + return session.commit( + async (tx) => { + const note = await tx.doc(Note, id); + return note.n++; + }, + context, + options, + ); +} + +/** Record the order in which `promises` settle, by name; a rejection as `name!`. */ +function settleOrder(promises: Record>): string[] { + const order: string[] = []; + for (const [name, promise] of Object.entries(promises)) { + promise.then( + () => order.push(name), + () => order.push(`${name}!`), + ); + } + return order; +} + +/** The note values a reopened Session over `storage` reads, per conversation. */ +async function notes(storage: BatchStorage, ids: readonly ConversationId[]): Promise { + const reopened = new SessionImpl(storage.reopen()); + const values: number[] = []; + for (const id of ids) values.push((await reopened.snapshot(Note, id, context))?.n ?? 0); + await reopened.close(context); + return values; +} + +/** The kinds of the entries `published` holds, in order. */ +function entryKinds(published: readonly ObservedCommit[]): string[] { + return published.flatMap((publication) => + publication.kind === "stored" + ? publication.changes.flatMap((change) => (change.type === "entry" ? [change.value.kind] : [])) + : [], + ); +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("group commit: batching", () => { + it("stores the commits admitted while a call is in flight in one call, each with its seq, published in order", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const ran: string[] = []; + const second = session.commit(async (tx) => { + (await tx.doc(Note, b)).n++; + ran.push("b"); + }, context); + const third = session.commit(async (tx) => { + (await tx.doc(Note, c)).n++; + ran.push("c"); + }, context); + // Their callbacks run while the first commit is in flight: they touch nothing it writes. + await eventually(() => ran.length === 2); + expect(published).toHaveLength(0); + const order = settleOrder({ a: first, b: second, c: third }); + held.release(); + await Promise.all([first, second, third]); + expect(order).toEqual(["a", "b", "c"]); + expect(storage.calls.slice(-2).map((call) => call.length)).toEqual([1, 2]); + const seqs = published.map((publication) => (publication.kind === "stored" ? publication.seq : 0)); + expect(seqs).toEqual([...seqs].sort((x, y) => x - y)); + expect(new Set(seqs).size).toBe(3); + await session.close(context); + }); +}); + +describe("group commit: gates", () => { + it("makes a read of an entry ID leaked by an unpublished commit wait for its publication", async () => { + const { storage, session, ids } = await open(1); + const held = storage.hold(); + let leaked: EntryId | undefined; + const appending = session.commit(async (tx) => { + leaked = (await tx.appendEntry(ids[0]!, { kind: "note" })).id; + }, context); + await held.entered; + const reading = session.commit((tx) => tx.entry(leaked!), context); + expect(await settled(reading)).toBe(false); + held.release(); + await appending; + expect((await reading)?.id).toBe(leaked); + await session.close(context); + }); + + it("makes a job wait for an unpublished creation at a document address instead of creating it again", async () => { + const { storage, session } = await open(0); + const held = storage.hold(); + const first = session.commit(async (tx) => { + (await tx.doc(Shared)).n++; + }, context); + await held.entered; + const second = session.commit(async (tx) => { + const shared = await tx.doc(Shared); + shared.n++; + return shared.n; + }, context); + expect(await settled(second)).toBe(false); + held.release(); + await first; + expect(await second).toBe(2); + const creations = storage.calls + .flat(2) + .filter((write) => write.type === "document.create" && write.record.kind === Shared.definition.kind); + expect(creations).toHaveLength(1); + await session.close(context); + }); + + it("makes a write to a conversation an unpublished commit creates wait for it", async () => { + const { storage, session } = await open(0); + const held = storage.hold(); + let leaked: ConversationId | undefined; + const creating = session.commit(async (tx) => { + leaked = (await tx.createConversation({ ownership: { kind: "ownerless" } })).id; + }, context); + await held.entered; + const appending = session.commit((tx) => tx.appendEntry(leaked!, { kind: "note" }), context); + expect(await settled(appending)).toBe(false); + held.release(); + await creating; + expect((await appending).conversationId).toBe(leaked); + await session.close(context); + }); + + it("makes each read of the entries of a conversation an unpublished commit creates wait for its creation", async () => { + // The Session's own helpers (newest entry, head marker), a transaction's scan, and the line's reads, each alone + // behind the creation, of a conversation Storage does not have yet. + const reads: Record Promise> = { + latestEntryId: (session, id) => session.commit((tx) => tx.latestEntryId(id), context), + latestHeadMarker: (session, id) => session.commit((tx) => tx.latestHeadMarker(id), context), + "tx.scanEntries": (session, id) => + session.commit(async (tx) => (await tx.scanEntries({ conversationId: id }, 1)).items, context), + "line scanEntries": (session, id) => + session.readOnLine( + async (line) => (await line.scanEntries({ conversationId: id }, 1, undefined, context)).items, + ), + "line findLatestHeadMarker": (session, id) => + session.readOnLine((line) => line.findLatestHeadMarker(id, undefined, context)), + }; + for (const [name, read] of Object.entries(reads)) { + const { storage, session } = await open(0); + const held = storage.hold(); + let leaked: ConversationId | undefined; + const creating = session.commit(async (tx) => { + leaked = (await tx.createConversation({ ownership: { kind: "ownerless" } })).id; + }, context); + await held.entered; + const reading = read(session, leaked!); + await flush(); + held.release(); + await creating; + const result = await reading.then( + (value) => ({ value }), + (error: unknown) => ({ error }), + ); + expect(result, name).toEqual({ value: Array.isArray((result as { value?: unknown }).value) ? [] : undefined }); + await session.close(context); + } + }); + + it("makes a scan that cannot be keyed wait for every unpublished commit", async () => { + const { storage, session } = await open(0); + const held = storage.hold(); + const creating = session.commit( + async (tx) => (await tx.createConversation({ ownership: { kind: "ownerless" } })).id, + context, + ); + await held.entered; + const scanning = session.commit( + async (tx) => (await tx.scanConversations({}, 10)).items.map((c) => c.id), + context, + ); + expect(await settled(scanning)).toBe(false); + held.release(); + expect(await scanning).toContain(await creating); + await session.close(context); + }); + + it("makes a line read after an un-awaited commit see it", async () => { + const { storage, session, ids } = await open(1); + const held = storage.hold(); + let leaked: EntryId | undefined; + void session.commit(async (tx) => { + leaked = (await tx.appendEntry(ids[0]!, { kind: "note" })).id; + }, context); + await held.entered; + const reading = session.readOnLine((storage) => storage.entry(leaked!, context)); + await flush(); + held.release(); + expect((await reading)?.entry.id).toBe(leaked); + await session.close(context); + }); + + it("makes a reader of a document whose held changes an unpublished commit stores wait, and store them once", async () => { + const { storage, session, ids } = await open(1); + const id = ids[0]!; + await bump(session, id); + await bump(session, id, DEFERRED); + // The first commit only reads the held document: storing its held changes is its write of it. + const held = storage.hold(); + const storing = session.commit(async (tx) => { + await tx.doc(Note, id); + await tx.appendEntry(id, { kind: "a" }); + }, context); + await held.entered; + let read = false; + const reading = session.commit(async (tx) => { + await tx.doc(Note, id); + read = true; + await tx.appendEntry(id, { kind: "b" }); + }, context); + await flush(); + expect(read).toBe(false); + held.release(); + await Promise.all([storing, reading]); + const changes = storage.calls + .slice(-2) + .map((call) => call.flat().filter((write) => write.type === "document.change")); + expect(changes.map((writes) => writes.length)).toEqual([1, 0]); + const record = (await storage.findDocument( + { kind: Note.definition.kind, scope: { kind: "conversation", conversationId: id } }, + "current", + context, + ))!; + expect((await storage.document(record.id, "current", context))?.deltasSinceBase).toBe(1); + await session.close(context); + }); + + it("makes a reader of a document held by a deferred commit behind a call wait for it, and store it", async () => { + const { storage, session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + const held = storage.hold(); + const first = bump(session, b); + await held.entered; + const deferredCommit = session.commit( + async (tx) => { + (await tx.doc(Note, a)).n = 5; + }, + context, + DEFERRED, + ); + const reader = bump(session, a); + await flush(); + // Nothing is published while the first commit is in flight, the deferred one neither. + expect(published).toHaveLength(1); + held.release(); + await Promise.all([first, deferredCommit]); + expect(await reader).toBe(5); + // The reader's own batch stored what the deferred commit held. + const stored = storage.calls.at(-1)!.flat(); + expect(stored.filter((write) => write.type === "document.change")).toHaveLength(1); + await session.close(context); + expect(await notes(storage, [a, b])).toEqual([6, 1]); + }); +}); + +describe("group commit: outcomes", () => { + it("delivers a read, a commit without writes, a callback error, and a deferred commit after an earlier commit", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const order = settleOrder({ + first, + read: session.readOnLine(async () => "read"), + none: session.commit(async () => "none", context), + error: session.commit(async () => { + throw new Error("own"); + }, context), + deferred: bump(session, b, DEFERRED), + }); + await flush(); + expect(order).toEqual([]); + held.release(); + await eventually(() => order.length === 5); + // A callback's own error affects only its job. + expect(order).toEqual(["first", "read", "none", "error!", "deferred"]); + await session.close(context); + }); + + it("makes every outcome after a fatal failure SessionFailed, and keeps the earlier ones", async () => { + const { storage, session, ids } = await open(4); + const [a, b, c, d] = ids as [ConversationId, ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const stored = bump(session, b); + const refused = session.commit((tx) => tx.appendEntry(c, { kind: "refuse" }), context); + const after = bump(session, d); + const read = session.readOnLine(async () => "read"); + storage.refuse = (writes) => writes.some((write) => write.type === "entry" && write.value.kind === "refuse"); + // Every callback has run, so the three are admitted while the first call is in flight. + await flush(); + held.release(); + expect(await first).toBe(0); + expect(await stored).toBe(0); + await expect(refused).rejects.toThrow("refused"); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + await expect(read).rejects.toBeInstanceOf(SessionFailed); + expect(await session.closed).toEqual({ kind: "failed", error: new Error("refused") }); + // One call stored the first two of the three batches it got; the refused one and the one after it are not stored. + expect(storage.calls.at(-1)).toHaveLength(3); + expect(await notes(storage, [a, b, d])).toEqual([1, 1, 0]); + }); +}); + +describe("group commit: failures per part", () => { + it("publishes the parts received before a call rejects, then rejects them with its error", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const second = bump(session, b); + const third = bump(session, c); + storage.rejectCall = storage.calls.length; + await flush(); + held.release(); + await first; + await expect(second).rejects.toThrow("connection lost"); + await expect(third).rejects.toThrow("connection lost"); + // Durable, so published. + expect(published).toHaveLength(3); + expect(await session.closed).toEqual({ kind: "failed", error: new Error("connection lost") }); + expect(await notes(storage, [a, b, c])).toEqual([1, 1, 1]); + }); + + it("fails every job after the first part of a call that rejects, except its other parts, with their storage error", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const first = session.commit((tx) => tx.appendEntry(id, { kind: "a" }), context); + await held.entered; + // Between the two parts of the failing call, a job without writes. + const before = session.commit((tx) => tx.appendEntry(id, { kind: "b" }), context); + const between = session.commit(async () => "no write", context); + const after = session.commit((tx) => tx.appendEntry(id, { kind: "d" }), context); + const outcomes = Promise.allSettled([first, before, between, after]); + await flush(); + storage.rejectCall = storage.calls.length; + held.release(); + const [a, b, n, d] = await outcomes; + expect(a.status).toBe("fulfilled"); + expect(b).toMatchObject({ status: "rejected", reason: new Error("connection lost") }); + expect((n as PromiseRejectedResult).reason).toBeInstanceOf(SessionFailed); + expect(d).toMatchObject({ status: "rejected", reason: new Error("connection lost") }); + // Both parts were received, so they were published before they rejected. + expect(storage.calls.at(-1)).toHaveLength(2); + expect(published).toHaveLength(3); + expect(await session.closed).toEqual({ kind: "failed", error: new Error("connection lost") }); + }); + + it("reports SessionFailed with the first error for a call that rejects after another failure failed the Session", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const inFlight = session.commit((tx) => tx.appendEntry(id, { kind: "a" }), context); + await held.entered; + // A later job's read fails first, and fails the Session; then the call in flight rejects with another error. + const first = new Error("first failure: read"); + vi.spyOn(storage, "task").mockRejectedValueOnce(first); + const reading = session.commit((tx) => tx.task(idFromNumber(999)), context); + reading.catch(() => {}); + await flush(); + expect(session.failure).toEqual({ error: first }); + storage.rejectCall = storage.calls.length - 1; + held.release(); + await expect(reading).rejects.toBe(first); + const outcome = await inFlight.then( + () => undefined, + (error: unknown) => error, + ); + expect(outcome).toBeInstanceOf(SessionFailed); + expect((outcome as SessionFailed).cause).toBe(first); + // Nothing of the call is published once the Session failed. + expect(published).toEqual([]); + expect(await session.closed).toEqual({ kind: "failed", error: first }); + }); + + for (const value of [undefined, null]) { + it(`rejects every part of a call with its rejection value, ${value} included`, async () => { + const { storage, session, ids } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const settle = (promise: Promise) => + promise.then( + () => ({ kind: "resolved" }), + (error: unknown) => ({ kind: "rejected", error }), + ); + const parts = [settle(bump(session, b)), settle(bump(session, c))]; + await flush(); + storage.rejection = { value }; + storage.rejectCall = storage.calls.length; + held.release(); + await first; + expect(await Promise.all(parts)).toEqual([ + { kind: "rejected", error: value }, + { kind: "rejected", error: value }, + ]); + expect(await session.closed).toEqual({ kind: "failed", error: value }); + }); + + it(`rejects a refused part with its refusal, ${value} included, and the parts after it with SessionFailed`, async () => { + const { storage, session, ids } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const refused = session + .commit((tx) => tx.appendEntry(b, { kind: "refuse" }), context) + .then( + () => ({ kind: "resolved" }), + (error: unknown) => ({ kind: "rejected", error }), + ); + const after = bump(session, c).then( + () => undefined, + (error: unknown) => error, + ); + await flush(); + storage.refusal = { value }; + storage.refuse = (writes) => writes.some((write) => write.type === "entry" && write.value.kind === "refuse"); + held.release(); + await first; + expect(await refused).toEqual({ kind: "rejected", error: value }); + expect(await after).toBeInstanceOf(SessionFailed); + expect(await session.closed).toEqual({ kind: "failed", error: value }); + }); + } + + it("leaves the parts after one whose adoption throws unpublished, rejected with SessionFailed", async () => { + const { storage, session, ids, published } = await open(4); + const [a, b, c, d] = ids as [ConversationId, ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const before = bump(session, b); + const failing = bump(session, c); + const after = bump(session, d); + const failure = new Error("adoption failed"); + const adopt = Transaction.prototype.adopt; + let adoptions = 0; + vi.spyOn(Transaction.prototype, "adopt").mockImplementation(function (this: Transaction, seq: Seq) { + if (++adoptions === 3) throw failure; + return adopt.call(this, seq); + }); + await flush(); + held.release(); + await first; + await before; + await expect(failing).rejects.toBe(failure); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + expect(published).toHaveLength(2); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + // All four are durable: reopening recovers the unpublished one. + vi.restoreAllMocks(); + expect(await notes(storage, [a, b, c, d])).toEqual([1, 1, 1, 1]); + }); +}); + +describe("group commit: a publication that throws", () => { + it("fails the Session for a deferred commit memory cannot adopt, and still closes", async () => { + const { session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + await bump(session, b); + published.length = 0; + const failure = new Error("held adoption failed"); + vi.spyOn(Transaction.prototype, "adoptHeld").mockImplementationOnce(() => { + throw failure; + }); + const held = bump(session, a, DEFERRED); + const after = bump(session, b); + await expect(held).rejects.toBe(failure); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + expect(published).toEqual([]); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + await session.close(context); + }); + + it("settles a deferred commit whose adoption throws, alone, and closes", async () => { + // Astra's: the only commit is the deferred one, on a storage that runs nothing else. + const storage = new ControlledStorage(); + const session = new SessionImpl(storage); + const id = await session.commit(async (tx) => { + const conversation = await tx.createConversation({ ownership: { kind: "ownerless" } }); + await tx.doc(Note, conversation.id); + return conversation.id; + }, context); + const failure = new Error("injected held adoption failure"); + vi.spyOn(Transaction.prototype, "adoptHeld").mockImplementationOnce(() => { + throw failure; + }); + await expect(bump(session, id, DEFERRED)).rejects.toBe(failure); + expect(session.failure).toEqual({ error: failure }); + await session.close(context); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + expect(session.lineSizes).toEqual({ outstanding: 0, unpublished: 0, reads: 0 }); + }); + + it("fails the Session for a throw in the read caches' write-through, and leaves the later parts unpublished", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const failing = bump(session, b); + const after = bump(session, c); + await flush(); + const failure = new Error("cache write failed"); + const write = CachedStorage.prototype.published; + let writes = 0; + vi.spyOn(CachedStorage.prototype, "published").mockImplementation(function (this: CachedStorage, batch) { + if (++writes === 2) throw failure; + return write.call(this, batch); + }); + held.release(); + await first; + await expect(failing).rejects.toBe(failure); + await expect(after).rejects.toBeInstanceOf(SessionFailed); + expect(published).toHaveLength(1); + expect(await session.closed).toEqual({ kind: "failed", error: failure }); + await session.close(context); + }); +}); + +describe("group commit: deferred commits", () => { + it("publishes a deferred commit only once the commit before it is published", async () => { + const { storage, session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + // Only a change of an existing document is held. + await bump(session, b); + published.length = 0; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const later = bump(session, b, DEFERRED); + await flush(); + expect(published).toHaveLength(0); + held.release(); + await Promise.all([first, later]); + expect(published.map((publication) => publication.kind)).toEqual(["stored", "memory"]); + await session.close(context); + }); + + it("never publishes a deferred commit behind a commit that fails", async () => { + const { storage, session, ids, published } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, b); + published.length = 0; + const held = storage.hold(); + const first = session.commit((tx) => tx.appendEntry(a, { kind: "refuse" }), context); + await held.entered; + const later = bump(session, b, DEFERRED); + storage.refuse = (writes) => writes.some((write) => write.type === "entry" && write.value.kind === "refuse"); + held.release(); + await expect(first).rejects.toThrow("refused"); + await expect(later).rejects.toBeInstanceOf(SessionFailed); + expect(published).toEqual([]); + }); +}); + +describe("group commit: publication", () => { + it("publishes the parts of one call a macrotask apart: what one resolves runs before the next is published", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const second = bump(session, b); + const third = bump(session, c); + let seenAtSecond: number | undefined; + void second.then(() => { + seenAtSecond = published.length; + }); + await flush(); + held.release(); + await Promise.all([first, second, third]); + expect(storage.calls.at(-1)).toHaveLength(2); + expect(seenAtSecond).toBe(2); + expect(published).toHaveLength(3); + await session.close(context); + }); +}); + +describe("group commit: the storage's publication scope", () => { + it("publishes each commit inside the storage's scope, the next only once one it runs later ran", async () => { + const { storage, session, ids, published } = await open(3); + const [a, b, c] = ids as [ConversationId, ConversationId, ConversationId]; + const scopes: string[] = []; + let depth = 0; + // Runs each publication a macrotask later, inside a scope of its own. + (storage as { publishing?: (publish: () => void, context: Context) => void }).publishing = (publish) => { + setTimeout(() => { + depth++; + scopes.push(`open ${published.length}`); + publish(); + scopes.push(`close ${published.length}`); + depth--; + }, 0); + }; + let inside: boolean[] = []; + session.followCommits(() => { + inside.push(depth === 1); + }); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const rest = [bump(session, b), bump(session, c)]; + await flush(); + inside = []; + held.release(); + await Promise.all([first, ...rest]); + expect(inside).toEqual([true, true, true]); + expect(scopes).toEqual(["open 0", "close 1", "open 1", "close 2", "open 2", "close 3"]); + await session.close(context); + }); + + it("reports a scope that throws instead of running the publication, and publishes the commit", async () => { + const { storage, session, ids, published } = await open(1); + const reports: unknown[] = []; + session.report = (error) => { + reports.push(error); + }; + (storage as { publishing?: (publish: () => void, context: Context) => void }).publishing = () => { + throw new Error("no scope"); + }; + expect(await bump(session, ids[0]!)).toBe(0); + expect(published).toHaveLength(1); + expect(reports).toEqual([new Error("no scope")]); + await session.close(context); + }); +}); + +describe("group commit: a failure between two publications", () => { + it("publishes no part of a call after a failure that a continuation of an earlier part caused", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const lead = session.commit((tx) => tx.appendEntry(id, { kind: "lead" }), context); + await held.entered; + const one = session.commit((tx) => tx.appendEntry(id, { kind: "one" }), context); + const two = session.commit((tx) => tx.appendEntry(id, { kind: "two" }), context); + two.catch(() => {}); + // What `one` resolves runs in the hop before `two` is published: a read there fails the Session. + const read = new Error("read failed between publications"); + vi.spyOn(storage, "task").mockRejectedValue(read); + const failing = one.then(() => session.commit((tx) => tx.task(idFromNumber(999)), context)); + failing.catch(() => {}); + await flush(); + held.release(); + await Promise.allSettled([lead, one, two, failing]); + expect(entryKinds(published)).toEqual(["lead", "one"]); + const outcome = await two.then( + () => undefined, + (error: unknown) => error, + ); + expect(outcome).toBeInstanceOf(SessionFailed); + expect((outcome as SessionFailed).cause).toBe(read); + }); + + it("publishes no part of a call after the Session fails in a continuation of an earlier part", async () => { + // Sol's: `one`'s continuation fails the Session itself. + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const held = storage.hold(); + const lead = session.commit((tx) => tx.appendEntry(id, { kind: "lead" }), context); + await held.entered; + const failure = new Error("failed between publications"); + const one = session.commit((tx) => tx.appendEntry(id, { kind: "one" }), context); + void one.then(() => session.fail(failure)); + const two = session.commit((tx) => tx.appendEntry(id, { kind: "two" }), context); + two.catch(() => {}); + await flush(); + held.release(); + await Promise.allSettled([lead, one, two]); + expect(entryKinds(published)).toEqual(["lead", "one"]); + await expect(two).rejects.toBeInstanceOf(SessionFailed); + }); + + it("checks the failure when a storage's scope runs a publication it deferred, not when it was asked to", async () => { + const { storage, session, ids, published } = await open(1); + const id = ids[0]!; + const failure = new Error("failed before a deferred publication ran"); + let calls = 0; + // The second publication's scope is entered before the failure; it runs the publication only after it. + (storage as { publishing?: (publish: () => void, context: Context) => void }).publishing = (publish) => { + if (++calls === 2) session.fail(failure); + setTimeout(publish, 0); + }; + const held = storage.hold(); + const lead = session.commit((tx) => tx.appendEntry(id, { kind: "lead" }), context); + await held.entered; + const one = session.commit((tx) => tx.appendEntry(id, { kind: "one" }), context); + one.catch(() => {}); + await flush(); + held.release(); + await Promise.allSettled([lead, one]); + expect(entryKinds(published)).toEqual(["lead"]); + await expect(one).rejects.toBeInstanceOf(SessionFailed); + }); +}); + +describe("group commit: upkeep through the writer", () => { + it("stores held changes for an unload after the call in flight", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + await bump(session, b, DEFERRED); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const unloading = session.unloadDocuments(); + expect(await settled(unloading)).toBe(false); + held.release(); + await Promise.all([first, unloading]); + expect(storage.calls.at(-1)!.flat()).toMatchObject([{ type: "document.change" }]); + expect(await session.snapshot(Note, b, context)).toEqual({ n: 1 }); + await session.close(context); + expect(await notes(storage, [a, b])).toEqual([2, 1]); + }); + + it("stores held changes for a version reload inside a callback after the call in flight", async () => { + const NoteV2 = defineDoc<{ n: number; migrated: true }>({ + checkpointWhen: () => false, + kind: "gc.note", + version: 2, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ n: 0, migrated: true }), + migrate: (value) => ({ ...(value as { n: number }), migrated: true }), + }); + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + await bump(session, b); + await bump(session, b, DEFERRED); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const reloading = session.commit(async (tx) => (await tx.doc(NoteV2, b)).n, context); + expect(await settled(reloading)).toBe(false); + held.release(); + await first; + // The reload found what memory held, stored first. + expect(await reloading).toBe(2); + await session.close(context); + }); + + it("stores held changes at their deadline after the call in flight", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + await bump(session, a); + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const later = bump(session, b, { persistence: { kind: "deferred", withinMs: 0 } }); + await flush(); + const calls = storage.calls.length; + held.release(); + await Promise.all([first, later]); + await eventually(() => storage.calls.length === calls + 1); + expect(storage.calls.at(-1)!.flat()).toMatchObject([{ type: "document.create" }]); + await session.close(context); + }); + + it("drains the calls in flight and stores held changes when it closes", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + const later = bump(session, b, DEFERRED); + await flush(); + const closing = session.close(context); + held.release(); + await Promise.all([first, later]); + await closing; + expect(await session.closed).toEqual({ kind: "closed" }); + expect(await notes(storage, [a, b])).toEqual([1, 1]); + }); +}); + +describe("group commit: backpressure and bounds", () => { + it("starts no job past 256 without a delivered outcome, and returns its line state to zero when idle", async () => { + const { storage, session, ids } = await open(2); + const [a, b] = ids as [ConversationId, ConversationId]; + const held = storage.hold(); + const first = bump(session, a); + await held.entered; + let ran = 0; + const appends: Promise[] = []; + for (let n = 0; n < 300; n++) { + appends.push( + session.commit(async (tx) => { + ran++; + await tx.appendEntry(b, { kind: "note" }); + }, context), + ); + } + await eventually(() => ran === 255); + await flush(); + expect(ran).toBe(255); + expect(session.lineSizes.outstanding).toBe(256); + held.release(); + await Promise.all([first, ...appends]); + expect(ran).toBe(300); + expect(session.lineSizes).toEqual({ outstanding: 0, unpublished: 0, reads: 0 }); + await session.close(context); + }); +}); + +describe("group commit: randomized interleavings", () => { + /** A small deterministic generator, so a failing seed reproduces. */ + function random(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1_103_515_245 + 12_345) % 2_147_483_648; + return state / 2_147_483_648; + }; + } + + for (const seed of [1, 7, 42, 1_234, 99_991]) { + it(`stores, publishes, and reads only what an order of the commits implies (seed ${seed})`, async () => { + const next = random(seed); + const { storage, session, ids } = await open(4); + const log = ids[0]!; + storage.latency = () => Math.floor(next() * 3); + const refuseAt = seed % 2 === 0 ? Math.floor(next() * 60) : -1; + storage.refuse = (writes) => + writes.some( + (write) => write.type === "entry" && (write.value.data as { index?: number })?.index === refuseAt, + ); + // Per commit, in line order: the notes it increments and the values it read. + const plans: ConversationId[][] = []; + const reads: (number[] | undefined)[] = []; + const outcomes: Promise[] = []; + for (let index = 0; index < 60; index++) { + const touched = ids.filter(() => next() < 0.4); + plans.push(touched); + const outcome = session.commit(async (tx) => { + const values: number[] = []; + for (const id of touched) values.push((await tx.doc(Note, id)).n++); + reads[index] = values; + await tx.appendEntry(log, { kind: "log", data: { index } }); + }, context); + outcome.catch(() => {}); + outcomes.push(outcome); + const pause = next(); + if (pause < 0.3) await outcome.catch(() => {}); + else if (pause < 0.5) await flush(); + } + const results = await Promise.allSettled(outcomes); + if (refuseAt >= 0) await session.closed; + else await session.close(context); + // Outcomes match storage: a commit resolved exactly when its log entry is stored. + const reopened = new SessionImpl(storage.reopen()); + const logged = new Set(); + const entries = await reopened.commit((tx) => tx.scanEntries({ conversationId: log }, 1_000), context); + for (const entry of entries.items) logged.add((entry.data as { index: number }).index); + const stored = results.map((result) => result.status === "fulfilled"); + for (const [index, ok] of stored.entries()) expect(logged.has(index), `commit ${index}`).toBe(ok); + // Each callback read every earlier stored increment of what it touched, and nothing else. + for (const [index, values] of reads.entries()) { + if (values === undefined) continue; + for (const [position, id] of plans[index]!.entries()) { + const expected = plans + .slice(0, index) + .filter((touched, earlier) => stored[earlier] && touched.includes(id)); + expect(values[position], `commit ${index} read of ${id}`).toBe(expected.length); + } + } + // Storage equals the stored commits replayed in order. + for (const id of ids) { + const increments = plans.filter((touched, index) => stored[index] && touched.includes(id)).length; + expect((await reopened.snapshot(Note, id, context))?.n ?? 0).toBe(increments); + } + expect(session.lineSizes).toEqual({ outstanding: 0, unpublished: 0, reads: 0 }); + await reopened.close(context); + }); + } +}); diff --git a/packages/durable/test/session-observers.test.ts b/packages/durable/test/session-observers.test.ts index 0dcbc324a..cdc847b78 100644 --- a/packages/durable/test/session-observers.test.ts +++ b/packages/durable/test/session-observers.test.ts @@ -84,7 +84,7 @@ describe("observeDoc", () => { await session.close(context); }); - it("starts at the committed value and advances inside adopt with each stored commit's ops", async () => { + it("starts at the committed value and advances at the publication of each stored commit with its ops", async () => { const { storage, session } = open(); await add(session); const { calls, observer } = recorder(storage); @@ -95,7 +95,7 @@ describe("observeDoc", () => { expect(calls).toEqual([ { advance: { kind: "value", value: { count: 3 }, ops: [["s", ["count"], 3]], retired: false }, - adopting: true, + adopting: false, }, ]); observation.detach(); @@ -146,10 +146,10 @@ describe("observeDoc", () => { (await tx.doc(ConversationCounter, conversationId)).count = 9; }, context); expect(calls).toEqual([ - { advance: { kind: "value", value: observation.value, ops: [], retired: true }, adopting: true }, + { advance: { kind: "value", value: observation.value, ops: [], retired: true }, adopting: false }, { advance: { kind: "value", value: { count: 9 }, ops: [["r", { count: 9 }]], retired: false }, - adopting: true, + adopting: false, }, ]); expect((calls[0] as { advance: Extract, { kind: "value" }> }).advance.value).toBe( @@ -173,13 +173,31 @@ describe("observeDoc", () => { expect(calls).toEqual([]); }); - it("ends once when the Session closes", async () => { + it("ends once when the Session closes, `failed` (S1: an adapter follows the next Session), a watch `ended`", async () => { const { storage, session } = open(); await add(session); const { calls, observer } = recorder(storage); await session.observeDoc(SessionCounter, observer, context); + const watch = (await session.watchDoc(SessionCounter, context))!; await session.close(context); - expect(calls).toEqual([{ end: { kind: "ended" } }]); + expect(calls).toEqual([{ end: { kind: "failed", error: new Error("session closed") } }]); + expect(await watch.closed).toEqual({ kind: "ended" }); + }); + + it("ends each close's observers with an error of that close, not one shared by every Session", async () => { + const errors: unknown[] = []; + for (let i = 0; i < 2; i++) { + const { session } = open(); + await add(session); + await session.observeDoc( + SessionCounter, + { advance: () => {}, end: (end) => errors.push(end.kind === "failed" ? end.error : end) }, + context, + ); + await session.close(context); + } + expect(errors).toEqual([new Error("session closed"), new Error("session closed")]); + expect(errors[0]).not.toBe(errors[1]); }); it("ends with the error that failed the Session", async () => { @@ -368,14 +386,18 @@ describe("observeCommits", () => { return { calls, observer }; } - it("gets every stored commit, then `ended` once the Session closed, after its last publication", async () => { + it("gets every stored commit, then `failed` once the Session closed (S1), after its last publication", async () => { const { session } = open(); const { calls, observer } = commits(); session.observeCommits(observer); await add(session); await add(session); await session.close(context); - expect(calls).toEqual([{ seq: expect.any(Number) }, { seq: expect.any(Number) }, { end: { kind: "ended" } }]); + expect(calls).toEqual([ + { seq: expect.any(Number) }, + { seq: expect.any(Number) }, + { end: { kind: "failed", error: new Error("session closed") } }, + ]); }); it("ends `failed` with the error that failed the Session", async () => { diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index f31e4b266..8fb1a015c 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -3,6 +3,7 @@ import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { type CommitChange, type CommitPublication, + type CommitResult, type ConversationId, type DocumentAddress, type DocumentCommitChange, @@ -13,6 +14,7 @@ import { MemoryStorage, type Seq, type Session, + type Storage, type StorageWrite, } from "@earendil-works/pi-durable"; import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; @@ -48,6 +50,24 @@ export type Gate = { release(): void; }; +/** + * Commits `batches` through `commit` up to the first one `refuses` matches, which it refuses with `error`: what a + * storage does whose own check refuses that batch. + */ +export async function refusingAt( + commit: Storage["commit"], + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + refuses: (writes: readonly StorageWrite[]) => boolean, + error: unknown, +): Promise { + const at = batches.findIndex(refuses); + if (at < 0) return commit(batches, receive, commitContext); + const result = await commit(batches.slice(0, at), receive, commitContext); + return result.stored < at ? result : { stored: at, refused: error }; +} + /** Memory storage with observable commits, held calls, and injected commit failures. */ export class ControlledStorage extends MemoryStorage { /** Exact borrowed batches admitted by Session. */ @@ -56,9 +76,9 @@ export class ControlledStorage extends MemoryStorage { readonly commits: (readonly StorageWrite[])[] = []; mintCount = 0; documentReadCount = 0; - /** Set while `adopt` runs, the storage's synchronous step: code that sees it set runs inside `Storage.commit()`'s adopt. */ + /** Set while a receipt runs, the storage's synchronous step: code that sees it set runs inside `commit()`'s `receive`. */ adopting = false; - /** Runs once, in the next commit's synchronous step right after `adopt` returned. */ + /** Runs once, in the next commit's synchronous step right after its first receipt returned. */ afterAdopt: (() => void) | undefined; #commitGate: { gate: Deferred; entered: Deferred } | undefined; #findGate: { gate: Deferred; entered: Deferred } | undefined; @@ -97,27 +117,33 @@ export class ControlledStorage extends MemoryStorage { held.gate.resolve(); } + /** + * Records and holds the call, then fails its first batch if a failure is set (a refusal: nothing stored), or commits + * it in memory, `adopting` set while each receipt runs and `afterAdopt` run after the first one. + */ override async commit( - writes: readonly StorageWrite[], - adopt: (seq: Seq) => void, + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, commitContext: Context, - ): Promise { - this.admittedCommits.push(writes); - this.commits.push(structuredClone(writes)); + ): Promise { const held = this.#commitGate; if (held !== undefined) { held.entered.resolve(); await held.gate.promise; } const failure = this.#commitFailure; + for (const writes of failure === undefined ? batches : batches.slice(0, 1)) { + this.admittedCommits.push(writes); + this.commits.push(structuredClone(writes)); + } if (failure !== undefined) { this.#commitFailure = undefined; - throw failure; + return { stored: 0, refused: failure }; } - const step = (seq: Seq): void => { + const step = (index: number, seq: Seq): void => { this.adopting = true; try { - adopt(seq); + receive(index, seq); } finally { this.adopting = false; } @@ -125,7 +151,7 @@ export class ControlledStorage extends MemoryStorage { this.afterAdopt = undefined; after?.(); }; - return super.commit(writes, step, commitContext); + return super.commit(batches, step, commitContext); } override mintId>(): Promise { @@ -164,21 +190,26 @@ export class LateResolvingStorage extends ControlledStorage { gap: ((writes: readonly StorageWrite[]) => boolean) | undefined; rejectAfterAdopt: ((writes: readonly StorageWrite[]) => boolean) | undefined; - override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) { + override async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { this.committing = true; try { - const seq = await super.commit(writes, adopt, commitContext); + const result = await super.commit(batches, receive, commitContext); + const stored = batches.slice(0, result.stored); await flush(); - if (this.gap?.(writes) === true) { + if (stored.some((writes) => this.gap?.(writes) === true)) { this.gap = undefined; // What the gap started runs before the commit resolves. await flush(); } - if (this.rejectAfterAdopt?.(writes) === true) { + if (stored.some((writes) => this.rejectAfterAdopt?.(writes) === true)) { this.rejectAfterAdopt = undefined; throw new Error("reclaim failed"); } - return seq; + return result; } finally { this.committing = false; } diff --git a/packages/durable/test/spec-usage.test.ts b/packages/durable/test/spec-usage.test.ts index c50cc255e..b1896bf66 100644 --- a/packages/durable/test/spec-usage.test.ts +++ b/packages/durable/test/spec-usage.test.ts @@ -3,8 +3,9 @@ * the names the spec leaves to the application declared below. Keep the two in sync; `test/examples/` runs the same * patterns end to end. */ + import type { Context, Draft } from "@earendil-works/chord"; -import { type AssistantMessage, type Models, type ToolCall, Type } from "@earendil-works/pi-ai"; +import type { AssistantMessage, Models, ToolCall } from "@earendil-works/pi-ai"; import { type Conversation, type ConversationId, @@ -34,6 +35,7 @@ import { type UserInput, wrapTool, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { expect, expectTypeOf, it } from "vitest"; import type { ExecutionEnv } from "../src/env/index.ts"; import { CodingTools, createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; @@ -169,7 +171,7 @@ function examples() { defineTool({ name: "subagent", description: "Delegate a self-contained task to a subagent and get its answer back.", - parameters: Type.Object({ task: Type.String() }), + parameters: obj({ task: str() }), replay: "safe", execute: async (args, api, context) => { const { task } = args; diff --git a/packages/durable/test/sqlite-cloudflare.test.ts b/packages/durable/test/sqlite-cloudflare.test.ts index e9580829f..14608a3b1 100644 --- a/packages/durable/test/sqlite-cloudflare.test.ts +++ b/packages/durable/test/sqlite-cloudflare.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it } from "vitest"; import { durableObjectStorage } from "../../chord/test/sqlite/helpers.ts"; import { idFromNumber } from "../src/ids.ts"; import { DurableObjectSqliteDatabase, openDurableObjectSqliteStorage } from "../src/storage/sqlite/cloudflare.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; registerStorageConformance({ describe, expect, it }, "SqliteStorage on a prefixed Durable Object", async (use) => @@ -48,20 +49,20 @@ describe("Durable Object SQLite adapter", () => { ]); const entryId = idFromNumber(2); await Promise.all([ - first.commit( + commitWrites( + first, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "first" } }, ], - () => {}, BACKGROUND_CONTEXT, ), - second.commit( + commitWrites( + second, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "second" } }, ], - () => {}, BACKGROUND_CONTEXT, ), ]); diff --git a/packages/durable/test/sqlite-facade.test.ts b/packages/durable/test/sqlite-facade.test.ts index 7ba878edb..d6c6995c0 100644 --- a/packages/durable/test/sqlite-facade.test.ts +++ b/packages/durable/test/sqlite-facade.test.ts @@ -11,11 +11,9 @@ import { idFromNumber } from "../src/ids.ts"; import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "../src/storage/sqlite/index.ts"; import { SqliteStorage } from "../src/storage/sqlite/index.ts"; import { NodeSqliteDatabase, openNodeSqliteDatabase } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { type DocumentId, type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - type SettlementMode = "immediate" | "delay" | "reject"; class ControlledSettlementDatabase implements SqliteDatabase { @@ -101,29 +99,29 @@ describe("portable SQLite facade settlement", () => { it("prepares each storage statement once per connection and reuses it across transactions", async () => { const connection = new PrepareCountingDatabaseSync(":memory:"); const storage = await SqliteStorage.open(new NodeSqliteDatabase(connection)); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT); - await storage.commit( + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT); + await commitWrites( + storage, Array.from({ length: 100 }, (_, index) => ({ type: "entry" as const, value: { id: idFromNumber(index + 2), conversationId: ROOT_CONVERSATION_ID, kind: "cached" }, })), - adopt, BACKGROUND_CONTEXT, ); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: { id: idFromNumber(102), conversationId: ROOT_CONVERSATION_ID, kind: "cached-again" }, }, ], - adopt, BACKGROUND_CONTEXT, ); expect((await storage.entry(idFromNumber(2), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached"); expect((await storage.entry(idFromNumber(102), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached-again"); await expect( - storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, BACKGROUND_CONTEXT), + commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], BACKGROUND_CONTEXT), ).rejects.toThrow("ID 1 already belongs to conversation"); expect((await storage.entry(idFromNumber(2), BACKGROUND_CONTEXT))?.entry.kind).toBe("cached"); expect(connection.repeatedPrepares()).toEqual([]); @@ -133,9 +131,9 @@ describe("portable SQLite facade settlement", () => { it("reads storage commits in call order", async () => { const database = await openNodeSqliteDatabase(":memory:"); const storage = await SqliteStorage.open(database); - const commit = storage.commit( + const commit = commitWrites( + storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], - adopt, BACKGROUND_CONTEXT, ); const read = storage.conversation(ROOT_CONVERSATION_ID, BACKGROUND_CONTEXT); @@ -147,12 +145,12 @@ describe("portable SQLite facade settlement", () => { it("lets admitted multi-query reads finish before storage closes", async () => { const storage = await SqliteStorage.open(await openNodeSqliteDatabase(":memory:")); const entryId = idFromNumber(2); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: { id: entryId, conversationId: ROOT_CONVERSATION_ID, kind: "probe" } }, ], - adopt, BACKGROUND_CONTEXT, ); const scan = storage.scanEntries({ conversationId: ROOT_CONVERSATION_ID }, 10, undefined, BACKGROUND_CONTEXT); @@ -184,14 +182,14 @@ describe("portable SQLite facade settlement", () => { const storage = await opening; database.controlNextSettlement("delay"); - const committing = storage.commit( + const committing = commitWrites( + storage, [ { type: "entry", value: { id: idFromNumber(100), conversationId: ROOT_CONVERSATION_ID, kind: "settled" }, }, ], - adopt, BACKGROUND_CONTEXT, ); let committed = false; @@ -206,14 +204,14 @@ describe("portable SQLite facade settlement", () => { expect(await storage.mintId()).toBe(101); database.controlNextSettlement("reject"); - const rejected = storage.commit( + const rejected = commitWrites( + storage, [ { type: "entry", value: { id: idFromNumber(200), conversationId: ROOT_CONVERSATION_ID, kind: "rejected" }, }, ], - adopt, BACKGROUND_CONTEXT, ); expect(await storage.mintId()).toBe(102); @@ -229,7 +227,8 @@ describe("portable SQLite facade settlement", () => { // Each yield count starts the commit at a different point of the read's record and revision queries. for (let yields = 0; yields < 16; yields++) { const storage = await SqliteStorage.open(await openNodeSqliteDatabase(":memory:")); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -237,14 +236,13 @@ describe("portable SQLite facade settlement", () => { content: { kind: "base", version: 1, value: { value: 1 } }, }, ], - adopt, BACKGROUND_CONTEXT, ); const read = storage.document(id, "current", BACKGROUND_CONTEXT); for (let index = 0; index < yields; index++) await Promise.resolve(); - const replace = storage.commit( + const replace = commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { value: 2 } } }], - adopt, BACKGROUND_CONTEXT, ); const [stored] = await Promise.all([read, replace]); diff --git a/packages/durable/test/sqlite-migrations.test.ts b/packages/durable/test/sqlite-migrations.test.ts index 2c33edc2c..64224808d 100644 --- a/packages/durable/test/sqlite-migrations.test.ts +++ b/packages/durable/test/sqlite-migrations.test.ts @@ -11,6 +11,7 @@ import { type SqliteMigration, } from "../src/storage/sqlite/index.ts"; import { openNodeSqliteDatabase, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import { type EntryId, ROOT_CONVERSATION_ID } from "../src/types.ts"; const directories = new Set(); @@ -115,7 +116,8 @@ describe("durable SQLite migrations", () => { it("rolls a failed migration back and preserves stored data for a successful retry", async () => { const path = await databasePath(); const storage = await openNodeSqliteStorage(path); - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { @@ -128,7 +130,6 @@ describe("durable SQLite migrations", () => { }, }, ], - () => {}, BACKGROUND_CONTEXT, ); await storage.close(BACKGROUND_CONTEXT); diff --git a/packages/durable/test/sqlite-storage.test.ts b/packages/durable/test/sqlite-storage.test.ts index 811862dec..369768245 100644 --- a/packages/durable/test/sqlite-storage.test.ts +++ b/packages/durable/test/sqlite-storage.test.ts @@ -4,12 +4,19 @@ import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import type { Context, JsonValue } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import type { SqliteDatabase } from "@earendil-works/chord/sqlite"; import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; import { afterEach, describe, expect, it } from "vitest"; import { idFromNumber } from "../src/ids.ts"; -import type { SqliteStorage } from "../src/storage/sqlite/index.ts"; -import { type NodeSqliteStorageOptions, openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { type SqliteStorage, SqliteStorage as SqliteStorageClass } from "../src/storage/sqlite/index.ts"; +import { + type NodeSqliteStorageOptions, + openNodeSqliteDatabase, + openNodeSqliteStorage, +} from "../src/storage/sqlite/node.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; import type { + CommitResult, ConversationId, DocumentCreate, DocumentId, @@ -24,9 +31,6 @@ import type { } from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; -/** Storage-level commits here have no in-memory owner to adopt them. */ -const adopt = (): void => {}; - const context = BACKGROUND_CONTEXT; const openStorages = new Set(); const tempDirectories = new Set(); @@ -60,13 +64,13 @@ class ReopeningStorage implements Storage { } async commit( - writes: readonly StorageWrite[], - adopt: Parameters[1], - commitContext: Parameters[2], - ): Promise { - if (this.closed) return this.current.commit(writes, adopt, commitContext); + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.closed) return this.current.commit(batches, receive, commitContext); try { - return await this.current.commit(writes, adopt, commitContext); + return await this.current.commit(batches, receive, commitContext); } finally { await this.current.close(context); this.current = await openNodeSqliteStorage(this.path, this.options); @@ -140,7 +144,7 @@ function entry(id: EntryId, conversationId: ConversationId, data?: JsonValue): E } async function createRoot(storage: Storage): Promise { - return storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + return commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); } function pendingTask(id: TaskId): TaskRecord { @@ -179,7 +183,7 @@ describe("Pico SqliteStorage", () => { expect(await createRoot(storage)).toBe(1); const entryId = await storage.mintId(); expect( - await storage.commit([{ type: "entry", value: entry(entryId, ROOT_CONVERSATION_ID) }], adopt, context), + await commitWrites(storage, [{ type: "entry", value: entry(entryId, ROOT_CONVERSATION_ID) }], context), ).toBe(2); await storage.close(context); openStorages.delete(storage); @@ -192,9 +196,9 @@ describe("Pico SqliteStorage", () => { }); expect(await reopened.mintId()).toBe(entryId + 1); expect( - await reopened.commit( + await commitWrites( + reopened, [{ type: "task", value: pendingTask(await reopened.mintId>()) }], - adopt, context, ), ).toBe(3); @@ -257,7 +261,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -265,7 +270,6 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { retained: true } }, }, ], - adopt, context, ); const database = new DatabaseSync(path); @@ -283,7 +287,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -291,10 +296,10 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { nested: { value: 1 }, rows: [] } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -306,10 +311,10 @@ describe("Pico SqliteStorage", () => { }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -325,10 +330,10 @@ describe("Pico SqliteStorage", () => { }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -336,7 +341,6 @@ describe("Pico SqliteStorage", () => { content: { kind: "delta", version: 1, ops: [["s", ["nested", "value"], 4]] }, }, ], - adopt, context, ); @@ -367,7 +371,8 @@ describe("Pico SqliteStorage", () => { const circular: { self?: unknown } = {}; circular.self = circular; await expect( - storage.commit( + commitWrites( + storage, [ { type: "entry", value: entry(transientId, ROOT_CONVERSATION_ID) }, { @@ -375,7 +380,6 @@ describe("Pico SqliteStorage", () => { value: { ...pendingTask(transientTaskId), input: circular as unknown as JsonValue }, }, ], - adopt, context, ), ).rejects.toThrow("circular structure"); @@ -383,7 +387,7 @@ describe("Pico SqliteStorage", () => { expect(await storage.task(transientTaskId, context)).toBeUndefined(); const committedId = await storage.mintId(); expect( - await storage.commit([{ type: "entry", value: entry(committedId, ROOT_CONVERSATION_ID) }], adopt, context), + await commitWrites(storage, [{ type: "entry", value: entry(committedId, ROOT_CONVERSATION_ID) }], context), ).toBe(2); const db = new DatabaseSync(path, { readOnly: true }); @@ -406,15 +410,16 @@ describe("Pico SqliteStorage", () => { history: "rewindable", fork: "asOf", } satisfies DocumentCreate; - const createdAt = await storage.commit( + const createdAt = await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { count: 0 } } }], - adopt, context, ); let ancientAt = createdAt; let recentAt = createdAt; for (let count = 1; count <= 40; count++) { - recentAt = await storage.commit( + recentAt = await commitWrites( + storage, [ { type: "document.change", @@ -425,7 +430,6 @@ describe("Pico SqliteStorage", () => { : { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, context, ); if (count === 5) ancientAt = recentAt; @@ -500,7 +504,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); const id = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -508,13 +513,12 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); for (let count = 1; count <= 10; count++) { - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] } }], - adopt, context, ); } @@ -530,13 +534,14 @@ describe("Pico SqliteStorage", () => { } finally { revisions.close(); } - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { count: 11 } } }], - adopt, context, ); expect(revisionCount(path, id)).toBe(1); - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -544,11 +549,10 @@ describe("Pico SqliteStorage", () => { content: { kind: "delta", version: 1, ops: [["r", { count: 12 }]] }, }, ], - adopt, context, ); expect(revisionCount(path, id)).toBe(2); - await storage.commit([{ type: "document.retire", id }], adopt, context); + await commitWrites(storage, [{ type: "document.retire", id }], context); expect(revisionCount(path, id)).toBe(0); }); @@ -556,7 +560,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage({ walAutoCheckpointPages: 1 }); await createRoot(storage); for (let index = 0; index < 20; index++) { - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -566,7 +571,6 @@ describe("Pico SqliteStorage", () => { }), }, ], - adopt, context, ); } @@ -588,7 +592,8 @@ describe("Pico SqliteStorage", () => { await createRoot(storage); const id = await storage.mintId(); const large = "x".repeat(512 * 1024); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -596,12 +601,11 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { text: large } }, }, ], - adopt, context, ); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { text: "small" } } }], - adopt, context, ); const before = new DatabaseSync(path, { readOnly: true }); @@ -614,9 +618,9 @@ describe("Pico SqliteStorage", () => { before.close(); } expect(freeAfterDelete).toBeGreaterThan(0); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "base", version: 1, value: { text: large } } }], - adopt, context, ); const after = new DatabaseSync(path, { readOnly: true }); @@ -634,7 +638,8 @@ describe("Pico SqliteStorage", () => { const { storage, path } = await createSqliteStorage(); await createRoot(storage); for (let index = 0; index < 100; index++) { - await storage.commit( + await commitWrites( + storage, [ { type: "entry", @@ -644,12 +649,12 @@ describe("Pico SqliteStorage", () => { }), }, ], - adopt, context, ); } const documentId = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -663,11 +668,11 @@ describe("Pico SqliteStorage", () => { content: { kind: "base", version: 1, value: { count: 0 } }, }, ], - adopt, context, ); for (let count = 1; count <= 100; count++) { - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", @@ -675,7 +680,6 @@ describe("Pico SqliteStorage", () => { content: { kind: "delta", version: 1, ops: [["s", ["count"], count]] }, }, ], - adopt, context, ); } @@ -684,3 +688,130 @@ describe("Pico SqliteStorage", () => { expect((await stat(path)).size).toBeLessThan(1024 * 1024); }); }); + +describe("SqliteStorage batches", () => { + for (const mode of ["undefined rejection", "synchronous throw"] as const) { + it(`rejects the whole call, the batches before it included, when a later batch's check read fails with a ${mode}`, async () => { + // Sol's: the first batch passes its checks and is written; the second batch's check read fails. + const directory = await mkdtemp(join(tmpdir(), "pi-durable-sqlite-")); + tempDirectories.add(directory); + const database = await openNodeSqliteDatabase(join(directory, "storage.sqlite")); + let failCheck = false; + let reads = 0; + const faulty: SqliteDatabase = { + exec: (sql) => database.exec(sql), + run: (sql, ...params) => database.run(sql, ...params), + runCounted: (sql, ...params) => database.runCounted(sql, ...params), + get: (sql, ...params) => database.get(sql, ...params), + all: (sql, ...params) => database.all(sql, ...params), + close: () => database.close(), + transaction: (callback) => + database.transaction((transaction) => + callback({ + exec: (sql) => transaction.exec(sql), + run: (sql, ...params) => transaction.run(sql, ...params), + runCounted: (sql, ...params) => transaction.runCounted(sql, ...params), + all: (sql, ...params) => transaction.all(sql, ...params), + get: (sql, ...params) => { + if (failCheck && sql.includes("record_type") && ++reads === 2) { + if (mode === "synchronous throw") throw new Error("sync driver failure"); + return Promise.reject(undefined); + } + return transaction.get(sql, ...params); + }, + }), + ), + }; + const storage = await SqliteStorageClass.open(faulty); + openStorages.add(storage); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const ids = [await storage.mintId(), await storage.mintId()]; + const batches: StorageWrite[][] = ids.map((id) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "note" } }, + ]); + failCheck = true; + const receipts: number[] = []; + const outcome = await storage + .commit(batches, (index) => receipts.push(index), context) + .then( + () => "resolved", + () => "rejected", + ); + failCheck = false; + expect(outcome).toBe("rejected"); + expect(receipts).toEqual([]); + expect(await storage.entry(ids[0]!, context)).toBeUndefined(); + }); + } + + /** How a check's read of record types fails in the database. */ + const failures: Record Promise> = { + "an Error rejection": () => Promise.reject(new Error("disk I/O error")), + "a synchronous throw": () => { + throw new Error("disk I/O error"); + }, + "a primitive rejection": () => Promise.reject("disk I/O error"), + "an undefined rejection": () => Promise.reject(undefined), + }; + for (const [name, fail] of Object.entries(failures)) { + it(`rejects a batch call whose check read fails in the database with ${name}, and refuses only for its own checks`, async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-sqlite-")); + tempDirectories.add(directory); + const database = await openNodeSqliteDatabase(join(directory, "storage.sqlite")); + // A driver failure in a check's read, armed for the next batch call. + let failCheck = false; + const faulty: SqliteDatabase = { + exec: (sql) => database.exec(sql), + run: (sql, ...params) => database.run(sql, ...params), + runCounted: (sql, ...params) => database.runCounted(sql, ...params), + get: (sql, ...params) => database.get(sql, ...params), + all: (sql, ...params) => database.all(sql, ...params), + close: () => database.close(), + transaction: (callback) => + database.transaction((transaction) => + callback({ + exec: (sql) => transaction.exec(sql), + run: (sql, ...params) => transaction.run(sql, ...params), + runCounted: (sql, ...params) => transaction.runCounted(sql, ...params), + all: (sql, ...params) => transaction.all(sql, ...params), + get: (sql, ...params) => { + if (failCheck && sql.includes("record_type")) return fail(); + return transaction.get(sql, ...params); + }, + }), + ), + }; + const storage = await SqliteStorageClass.open(faulty); + openStorages.add(storage); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + const first = await storage.mintId(); + const second = await storage.mintId(); + const batches: StorageWrite[][] = [first, second].map((id) => [ + { type: "entry", value: { id, conversationId: ROOT_CONVERSATION_ID, kind: "note" } }, + ]); + failCheck = true; + const receipts: number[] = []; + const outcome = await storage + .commit(batches, (index) => receipts.push(index), context) + .then( + (result) => ({ kind: "resolved", result }), + () => ({ kind: "rejected" }), + ); + failCheck = false; + // Unknown outcome, rolled back whole: nothing received, nothing stored. + expect(outcome).toEqual({ kind: "rejected" }); + expect(receipts).toEqual([]); + expect(await storage.entry(first, context)).toBeUndefined(); + // Durable's own check still refuses, keeping the batches before it. + const refused = await storage.commit( + [batches[0]!, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }]], + (index) => receipts.push(index), + context, + ); + expect(refused.stored).toBe(1); + expect(String(refused.refused)).toContain("already belongs to conversation"); + expect(receipts).toEqual([0]); + expect(await storage.entry(first, context)).toBeDefined(); + }); + } +}); diff --git a/packages/durable/test/store-storage-commit.test.ts b/packages/durable/test/store-storage-commit.test.ts new file mode 100644 index 000000000..30f79a573 --- /dev/null +++ b/packages/durable/test/store-storage-commit.test.ts @@ -0,0 +1,769 @@ +// StoreStorage's commit path (`Storage.commit` over `ChordStore.commitSeries`), p3's primary storage: per call one +// backend transaction with one seq per batch; Durable's checks of a batch refuse it in its turn, the batches before it +// stored; a failed statement rolls the call back and rejects, the store going on; a failed `COMMIT` stops the store; +// receipts after `COMMIT`; each commit published in a host batch of its own; other users of the store wait their turn. + +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { BACKGROUND_CONTEXT, withTelemetry } from "@earendil-works/chord/context"; +import { type ChordStore, createHost, openStore, sqliteBackend } from "@earendil-works/chord/next"; +import type { SqliteDatabase, SqliteExecutor } from "@earendil-works/chord/sqlite"; +import { openNodeSqliteDatabase } from "@earendil-works/chord/sqlite/node"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { afterEach, describe, expect, it } from "vitest"; +import { idFromNumber } from "../src/ids.ts"; +import { MemoryStorage } from "../src/index.ts"; +import { SessionImpl } from "../src/session/session.ts"; +import { StoreStorage } from "../src/storage/store/index.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import type { + CommitResult, + ConversationId, + DocumentCreate, + DocumentId, + EntryId, + Seq, + Storage, + StorageWrite, +} from "../src/types.ts"; +import { ROOT_CONVERSATION_ID } from "../src/types.ts"; +import { openStoreStorage } from "./store-storage-support.ts"; + +const context = BACKGROUND_CONTEXT; + +let directory: string | undefined; + +afterEach(async () => { + if (directory !== undefined) await rm(directory, { recursive: true, force: true }); + directory = undefined; +}); + +/** A fresh SQLite file in this test's directory. */ +async function storePath(): Promise { + directory ??= await mkdtemp(join(tmpdir(), "pi-durable-store-commit-")); + return join(directory, `store-${Math.random().toString(36).slice(2)}.sqlite`); +} + +/** + * What the next backend transaction does besides its work: `commit` fails it after its work (a failed `COMMIT`, an + * unknown outcome); `fail` fails a statement it matches (by its SQL and its index, from 1), as a backend error; `hold` + * holds a statement until the promise it returns settles. + */ +type Faults = { + commit: boolean; + fail: ((sql: string, index: number) => boolean) | undefined; + hold: ((sql: string, index: number) => Promise | undefined) | undefined; +}; + +/** + * A storage with its root conversation, over a store with an outbox on the SQLite file at `path`, whose backend counts + * its transactions and the statements of the last one, and takes `faults` for its next transaction. + */ +async function counted(path = ":memory:") { + const db = await openNodeSqliteDatabase(path); + const counts = { transactions: 0, statements: 0 }; + const faults: Faults = { commit: false, fail: undefined, hold: undefined }; + const backend: SqliteDatabase = { + exec: (sql) => db.exec(sql), + run: (sql, ...params) => db.run(sql, ...params), + runCounted: (sql, ...params) => db.runCounted(sql, ...params), + get: (sql, ...params) => db.get(sql, ...params), + all: (sql, ...params) => db.all(sql, ...params), + close: () => db.close(), + transaction: (work) => { + counts.transactions++; + counts.statements = 0; + const { commit, fail, hold } = faults; + faults.commit = false; + faults.fail = undefined; + faults.hold = undefined; + return db.transaction(async (executor) => { + const call = async (sql: string, run: () => Promise): Promise => { + const index = ++counts.statements; + await hold?.(sql, index); + if (fail?.(sql, index) === true) throw new Error(`injected: statement ${index} failed`); + return run(); + }; + const result = await work({ + exec: (sql) => call(sql, () => executor.exec(sql)), + run: (sql, ...params) => call(sql, () => executor.run(sql, ...params)), + runCounted: (sql, ...params) => call(sql, () => executor.runCounted(sql, ...params)), + get: (sql, ...params) => call(sql, () => executor.get(sql, ...params)), + all: (sql, ...params) => call(sql, () => executor.all(sql, ...params)), + } satisfies SqliteExecutor); + if (commit) throw new Error("COMMIT failed"); + return result; + }); + }, + }; + const store = (await openStore(sqliteBackend(backend), context, { outbox: true })) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + return { store, storage, counts, faults }; +} + +/** The same storage opened again on the file at `path`, as another process would. */ +async function reopen(path: string) { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context)) as ChordStore; + return { store, storage: await StoreStorage.open(store, "harness", context) }; +} + +const entryWrite = (id: EntryId, kind = "m"): StorageWrite => ({ + type: "entry", + value: { id, conversationId: ROOT_CONVERSATION_ID, kind }, +}); + +/** One macrotask: what can run before it has run. */ +const macrotask = (): Promise => new Promise((resolve) => setTimeout(resolve, 0)); + +/** The store's outbox commits, by seq. */ +async function outboxSeqs(store: ChordStore): Promise { + const rows = await store.read((executor) => executor.all<{ seq: number }>("SELECT seq FROM chord_outbox"), context); + return rows.map((row) => row.seq); +} + +describe("one transaction per call", () => { + it("stores the batches in one backend transaction, one seq and one outbox commit each, receipts in order", async () => { + const { store, storage, counts } = await counted(); + const ids = [await storage.mintId(), await storage.mintId(), await storage.mintId()]; + const before = counts.transactions; + const last = store.lastSeq; + const receipts: [number, Seq][] = []; + const result = await storage.commit( + ids.map((id) => [entryWrite(id)]), + (index, seq) => receipts.push([index, seq]), + context, + ); + expect(result).toEqual({ stored: 3 }); + expect(counts.transactions - before).toBe(1); + expect(receipts).toEqual([ + [0, last + 1], + [1, last + 2], + [2, last + 3], + ]); + for (const [index, id] of ids.entries()) { + expect((await storage.entry(id, context))?.commitSeq).toBe(last + 1 + index); + } + const outbox = await store.read( + (executor) => executor.all<{ seq: number }>("SELECT seq FROM chord_outbox WHERE seq > ?", last), + context, + ); + expect(outbox.map((row) => row.seq)).toEqual([last + 1, last + 2, last + 3]); + await store.close(context); + }); + + it("writes the metadata once per call: three statements per entry append, plus one (cost guard)", async () => { + const { store, storage, counts } = await counted(); + const telemetry = new InMemoryTelemetryContext(); + const span = await telemetry.startSpan({ name: "test" }, (each) => each); + const metered = withTelemetry(span, context); + const statements = (): number => + telemetry + .getMetrics() + .filter(({ name }) => name === "chord.store.statements") + .reduce((total, { value }) => total + value, 0); + // Alone, an append is its id check, its row and claim, its outbox commit, and the metadata. + await commitWrites(storage, [entryWrite(await storage.mintId())], metered); + expect(statements()).toBe(5); + const ids = [await storage.mintId(), await storage.mintId(), await storage.mintId()]; + const before = counts.transactions; + await storage.commit( + ids.map((id) => [entryWrite(id)]), + () => {}, + metered, + ); + expect([counts.transactions - before, statements() - 5]).toEqual([1, 3 * 4 + 1]); + await store.close(context); + }); +}); + +describe("refusals: the batches before the refused one stand", () => { + it("stores the batches before a refused one with Durable's message, none after it, and goes on", async () => { + const { store, storage } = await counted(); + const first = await storage.mintId(); + const after = await storage.mintId(); + const last = store.lastSeq; + const receipts: number[] = []; + const result = await storage.commit( + [[entryWrite(first)], [entryWrite(first, "again")], [entryWrite(after)]], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: ID ${first} already belongs to entry`); + expect(receipts).toEqual([0]); + expect((await storage.entry(first, context))?.entry.kind).toBe("m"); + expect(await storage.entry(after, context)).toBeUndefined(); + expect(store.lastSeq).toBe(last + 1); + expect(await commitWrites(storage, [entryWrite(after)], context)).toBe(last + 2); + await store.close(context); + }); + + it("refuses a batch its staging fails in that batch's turn, keeping the batches before it (review)", async () => { + const { store, storage } = await counted(); + const entry = await storage.mintId(); + const document = await storage.mintId(); + const receipts: number[] = []; + const result = await storage.commit( + [ + [entryWrite(entry)], + // Staging refuses it: one document retired twice. + [ + { type: "document.retire", id: document }, + { type: "document.retire", id: document }, + ], + ], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toBe(`Error: Document ${document} is retired more than once`); + expect(receipts).toEqual([0]); + expect(await storage.entry(entry, context)).toBeDefined(); + await store.close(context); + }); + + it.each([ + [ + "a duplicate retirement", + [ + { type: "document.retire", id: 101 }, + { type: "document.retire", id: 101 }, + ], + ], + [ + "a duplicate content command", + [ + { type: "document.change", id: 101, content: { kind: "base", version: 1, value: {} } }, + { type: "document.change", id: 101, content: { kind: "base", version: 1, value: {} } }, + ], + ], + ] as const)( + "refuses a later batch with %s as Memory does, keeping the prefix (review)", + async (_name, malformed) => { + const batches: StorageWrite[][] = [ + [{ type: "conversation", value: { id: idFromNumber(100) } }], + malformed.map((write) => ({ ...write, id: idFromNumber(write.id) })) as StorageWrite[], + ]; + const outcomes = []; + for (const storage of [new MemoryStorage(), await openStoreStorage(":memory:")]) { + const receipts: number[] = []; + const result = await storage.commit(batches, (index) => receipts.push(index), context); + const conversation = await storage.conversation(idFromNumber(100), context); + outcomes.push({ stored: result.stored, refused: String(result.refused), receipts, conversation }); + await storage.close(context); + } + expect(outcomes[0]).toMatchObject({ stored: 1, receipts: [0], conversation: { id: 100 } }); + expect(outcomes[1]).toEqual(outcomes[0]); + }, + ); + + it.each([ + [ + "a change of a document an earlier batch retired", + (id: DocumentId): StorageWrite[][] => [ + [{ type: "document.retire", id }], + [{ type: "document.change", id, content: { kind: "base", version: 1, value: { n: 2 } } }], + ], + "is retired", + ], + [ + "a creation at the address an earlier batch took", + (id: DocumentId): StorageWrite[][] => [ + [ + { + type: "document.create", + record: { id: idFromNumber(id + 1), kind: "n", key: "taken", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 2 } }, + }, + ], + [ + { + type: "document.create", + record: { id: idFromNumber(id + 2), kind: "n", key: "taken", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 3 } }, + }, + ], + ], + "already has a current incarnation", + ], + ] as const)("refuses %s against the state the batches before it left", async (_name, calls, message) => { + const { store, storage } = await counted(); + const document = idFromNumber(200); + await commitWrites( + storage, + [ + { + type: "document.create", + record: { id: document, kind: "n", scope: { kind: "session" } }, + content: { kind: "base", version: 1, value: { n: 1 } }, + }, + ], + context, + ); + const last = store.lastSeq; + const receipts: number[] = []; + const result = await storage.commit(calls(document), (index) => receipts.push(index), context); + expect(result.stored).toBe(1); + expect(String(result.refused)).toContain(message); + expect([receipts, store.lastSeq]).toEqual([[0], last + 1]); + expect((await outboxSeqs(store)).at(-1)).toBe(last + 1); + await store.close(context); + }); + + it("refuses a batch whose rows staging cannot encode, keeping the batches before it", async () => { + const { store, storage } = await counted(); + const entry = await storage.mintId(); + const receipts: number[] = []; + // No integer: the Session never sends one (Storage trusts it); the store's columns refuse it at staging. + const result = await storage.commit( + [[entryWrite(entry)], [entryWrite(idFromNumber(1.5))]], + (index) => receipts.push(index), + context, + ); + expect(result.stored).toBe(1); + expect(String(result.refused)).toContain('column "id": 1.5 is not a int value'); + expect(receipts).toEqual([0]); + expect(await storage.entry(entry, context)).toBeDefined(); + await store.close(context); + }); + + it("checks a batch against the batches before it in the call: a copy reads their state", async () => { + const { store, storage } = await counted(); + const source = await storage.mintId(); + const child = await storage.mintId(); + const copy = await storage.mintId(); + const record = (id: DocumentId, conversationId: ConversationId) => + ({ + id, + kind: "c", + scope: { kind: "conversation", conversationId }, + history: "rewindable", + fork: "asOf", + }) as const; + const result = await storage.commit( + [ + [ + { + type: "document.create", + record: record(source, ROOT_CONVERSATION_ID), + content: { kind: "base", version: 1, value: { n: 1 } }, + }, + ], + [{ type: "document.change", id: source, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }], + [ + { type: "conversation", value: { id: child } }, + { type: "document.copy", record: record(copy, child), source: { id: source, at: "current" } }, + ], + ], + () => {}, + context, + ); + expect(result).toEqual({ stored: 3 }); + expect((await storage.document(copy, "current", context))?.value).toEqual({ n: 2 }); + await store.close(context); + }); +}); + +describe("failures: a rollback rejects, an unknown outcome stops the store", () => { + it("rejects with nothing stored when a batch's check fails in the backend, and the store goes on", async () => { + const { store, storage, faults } = await counted(); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const last = store.lastSeq; + // The second batch's id check: its first read of the record ids. + let claimReads = 0; + faults.fail = (sql) => sql.includes("record_ids") && ++claimReads === 2; + const receipts: number[] = []; + await expect( + storage.commit([[entryWrite(first)], [entryWrite(second)]], (index) => receipts.push(index), context), + ).rejects.toThrow("injected"); + expect([receipts, store.lastSeq]).toEqual([[], last]); + expect(await storage.entry(first, context)).toBeUndefined(); + expect(await commitWrites(storage, [entryWrite(first)], context)).toBe(last + 1); + await store.close(context); + }); + + it("a failed COMMIT is an unknown outcome: the call rejects storage_failed, nothing is received or stored", async () => { + const path = await storePath(); + const { store, storage, faults } = await counted(path); + const ids = [await storage.mintId(), await storage.mintId()]; + const last = store.lastSeq; + faults.commit = true; + const receipts: number[] = []; + await expect( + storage.commit( + ids.map((id) => [entryWrite(id)]), + (index) => receipts.push(index), + context, + ), + ).rejects.toMatchObject({ code: "unavailable", reason: "storage_failed" }); + expect(receipts).toEqual([]); + await expect(storage.entry(ids[0]!, context)).rejects.toMatchObject({ reason: "storage_failed" }); + await store.close(context); + const reopened = await reopen(path); + expect(await reopened.storage.entry(ids[0]!, context)).toBeUndefined(); + expect(await commitWrites(reopened.storage, [entryWrite(ids[1]!)], context)).toBe(last + 1); + await reopened.store.close(context); + }); + + /** The record the audit's source document and its copy share. */ + const record = (id: number, conversationId: number): DocumentCreate => ({ + id: idFromNumber(id), + kind: "audit", + scope: { kind: "conversation", conversationId: idFromNumber(conversationId) }, + history: "rewindable", + fork: "asOf", + }); + + /** The source document, stored before the call. */ + const SEED: StorageWrite[] = [ + { + type: "document.create", + record: record(100, ROOT_CONVERSATION_ID), + content: { kind: "base", version: 1, value: { n: 0 } }, + }, + ]; + + /** A delta of the source document setting `n`. */ + const delta = (n: number): StorageWrite => ({ + type: "document.change", + id: idFromNumber(100), + content: { kind: "delta", version: 1, ops: [["s", ["n"], n]] }, + }); + + /** + * Three batches: an entry and a delta; a conversation, a copy of the source as the first batch left it and an + * entry; then an entry and a delta, or, `refused`, an entry whose id the first batch took. + */ + const batches = (third: "valid" | "refused"): StorageWrite[][] => [ + [entryWrite(idFromNumber(101), "b1"), delta(1)], + [ + { type: "conversation", value: { id: idFromNumber(105) } }, + { + type: "document.copy", + record: record(102, 105), + source: { id: idFromNumber(100), at: "current" }, + }, + entryWrite(idFromNumber(103), "b2"), + ], + third === "valid" + ? [entryWrite(idFromNumber(104), "b3"), delta(2)] + : [entryWrite(idFromNumber(101), "again")], + ]; + + /** What a storage holds of the audit's records. */ + async function stateOf(storage: Storage) { + const entries: (string | null)[] = []; + for (const id of [101, 103, 104]) + entries.push((await storage.entry(idFromNumber(id), context))?.entry.kind ?? null); + return { + entries, + source: (await storage.document(idFromNumber(100), "current", context))?.value ?? null, + copy: (await storage.document(idFromNumber(102), "current", context))?.value ?? null, + child: (await storage.conversation(idFromNumber(105), context)) ?? null, + }; + } + + /** What a call returned and what it left: its result, its receipts' seqs after the store's last one, the state. */ + async function outcomeOf(storage: Storage, third: "valid" | "refused", last: number) { + const seqs: number[] = []; + const result: CommitResult = await storage.commit( + batches(third), + (_index, seq) => seqs.push(seq - last), + context, + ); + return { stored: result.stored, refused: String(result.refused), seqs, state: await stateOf(storage) }; + } + + /** Every error in `error`'s cause chain, as text. */ + function chain(error: unknown): string[] { + const messages: string[] = []; + for (let at = error; at instanceof Error; at = at.cause) messages.push(at.message); + return messages; + } + + it.each(["valid", "refused"] as const)( + "rolls a 3-batch call back at a failure of any of its statements, as at a failed COMMIT, and reopens before it (third %s)", + async (third) => { + // What MemoryStorage makes of the call: the outcome every retry must reach. + const memory = new MemoryStorage(); + await commitWrites(memory, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); + await commitWrites(memory, SEED, context); + const expected = await outcomeOf(memory, third, 2); + expect(expected).toMatchObject( + third === "valid" ? { stored: 3, seqs: [1, 2, 3] } : { stored: 2, seqs: [1, 2] }, + ); + // The statements of the call's transaction (a read transaction after it would count its own). + const probe = await counted(); + await commitWrites(probe.storage, SEED, context); + const result = await probe.storage.commit(batches(third), () => {}, context); + const total = probe.counts.statements; + // Its cost: no read beyond a batch's checks, the metadata once (statement counts per batch in the cost guard). + expect(total).toBe(third === "valid" ? 29 : 23); + expect(result.stored).toBe(expected.stored); + expect(await stateOf(probe.storage)).toEqual(expected.state); + await probe.store.close(context); + // Each statement in turn, then the COMMIT. + for (let at = 1; at <= total + 1; at++) { + const path = await storePath(); + const opened = await counted(path); + await commitWrites(opened.storage, SEED, context); + const last = opened.store.lastSeq; + const outbox = await outboxSeqs(opened.store); + const before = await stateOf(opened.storage); + if (at <= total) opened.faults.fail = (_sql, index) => index === at; + else opened.faults.commit = true; + const receipts: number[] = []; + const error = await opened.storage + .commit(batches(third), (each) => receipts.push(each), context) + .then( + () => undefined, + (rejection: unknown) => rejection, + ); + expect(receipts).toEqual([]); + expect(opened.store.lastSeq).toBe(last); + if (at <= total) { + // A rollback: the call rejects with its failure, the store goes on. + expect(chain(error)).toContain(`injected: statement ${at} failed`); + expect(await stateOf(opened.storage)).toEqual(before); + } else { + expect(error).toMatchObject({ code: "unavailable", reason: "storage_failed" }); + } + await opened.store.close(context); + // On disk: as before the call, the seq, the outbox, the records and the allocator. + const again = await reopen(path); + expect(again.store.lastSeq).toBe(last); + expect(await outboxSeqs(again.store)).toEqual(outbox); + expect(await stateOf(again.storage)).toEqual(before); + expect(await again.storage.mintId()).toBeGreaterThan(100); + // A retry reaches MemoryStorage's outcome. + expect(await outcomeOf(again.storage, third, last)).toEqual(expected); + expect(await outboxSeqs(again.store)).toEqual([...outbox, ...expected.seqs.map((seq) => last + seq)]); + await again.store.close(context); + } + }, + ); + + it("a call on a closed storage, or on a stopped store, rejects and stores nothing", async () => { + const { store, storage, faults } = await counted(); + const entry = await storage.mintId(); + faults.commit = true; + await expect(commitWrites(storage, [entryWrite(entry)], context)).rejects.toMatchObject({ + reason: "storage_failed", + }); + await expect(storage.commit([[entryWrite(entry)]], () => {}, context)).rejects.toMatchObject({ + reason: "storage_failed", + }); + await store.close(context); + const other = await counted(); + await other.storage.close(context); + await expect(other.storage.commit([[entryWrite(entry)]], () => {}, context)).rejects.toThrow("closed"); + await other.store.close(context); + }); + + it("a call queued when the store closes rejects released, and stores nothing; the running one is stored", async () => { + const path = await storePath(); + const { store, storage, faults } = await counted(path); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + faults.hold = (_sql, index) => { + if (index !== 1) return undefined; + entered.resolve(); + return release.promise; + }; + const running = storage.commit([[entryWrite(first)]], () => {}, context); + await entered.promise; + const queued = storage.commit([[entryWrite(second)]], () => {}, context); + const closing = store.close(context); + release.resolve(); + expect(await running).toEqual({ stored: 1 }); + await expect(queued).rejects.toMatchObject({ code: "released" }); + await closing; + const again = await reopen(path); + expect(await again.storage.entry(first, context)).toBeDefined(); + expect(await again.storage.entry(second, context)).toBeUndefined(); + await again.store.close(context); + }); +}); + +describe("durability", () => { + it("keeps every batch of a stored call across a crash: reopened, the seqs, rows, documents and ids are there", async () => { + const path = await storePath(); + const { store, storage } = await counted(path); + const entry = await storage.mintId(); + const document = await storage.mintId(); + const record = { + id: document, + kind: "n", + scope: { kind: "conversation", conversationId: ROOT_CONVERSATION_ID }, + history: "rewindable", + fork: "asOf", + } as const; + const seqs: Seq[] = []; + await storage.commit( + [ + [entryWrite(entry)], + [{ type: "document.create", record, content: { kind: "base", version: 1, value: { n: 1 } } }], + [{ type: "document.change", id: document, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }], + ], + (_index, seq) => seqs.push(seq), + context, + ); + const minted = await storage.mintId(); + // A crash: the store is never closed; another process opens the file. + const { store: other, storage: reopened } = await reopen(path); + expect((await reopened.entry(entry, context))?.commitSeq).toBe(seqs[0]); + expect((await reopened.document(document, seqs[1]!, context))?.value).toEqual({ n: 1 }); + expect((await reopened.document(document, "current", context))?.value).toEqual({ n: 2 }); + expect(await reopened.mintId()).toBeGreaterThan(document); + expect(await commitWrites(reopened, [entryWrite(minted)], context)).toBe(seqs[2]! + 1); + await other.close(context); + await store.close(context); + }); +}); + +describe("publication", () => { + it("publishes each commit of a call in a host batch of its own, after the call's one transaction", async () => { + const { store, storage, counts } = await counted(); + const host = createHost({ store, onError: () => {} }); + const session = new SessionImpl(storage); + const events: string[] = []; + session.followCommits((commit) => { + if (commit.kind !== "stored") return; + const batch = store.batch!; + events.push(`publish ${commit.seq} running=${batch.running}`); + batch.flushing(() => events.push(`flush ${commit.seq}`)); + }); + // Hold the first call, so the commits after it are sent together in the next one. + const calls: number[] = []; + const gate = Promise.withResolvers(); + const commit = storage.commit.bind(storage); + storage.commit = async (batches, receive, ctx) => { + calls.push(batches.length); + if (calls.length === 1) await gate.promise; + return commit(batches, receive, ctx); + }; + const first = session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "a" }), context); + await new Promise((resolve) => setTimeout(resolve, 0)); + const rest = [1, 2, 3].map((n) => + session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: `b${n}` }), context), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + const before = counts.transactions; + gate.resolve(); + await Promise.all([first, ...rest]); + expect(calls).toEqual([1, 3]); + expect(counts.transactions - before).toBe(2); + const seq = store.lastSeq; + expect(events).toEqual( + [seq - 3, seq - 2, seq - 1, seq].flatMap((each) => [`publish ${each} running=true`, `flush ${each}`]), + ); + await session.close(context); + await host.close(context); + await store.close(context); + }); +}); + +describe("other users of the store", () => { + it("another owner's call waits for a series in its transaction; the series' receipts come first", async () => { + const { store, storage, faults } = await counted(); + const other = await StoreStorage.open(store, "other", context); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + faults.hold = (_sql, index) => { + if (index !== 2) return undefined; + entered.resolve(); + return release.promise; + }; + const events: string[] = []; + const series = storage.commit( + [[entryWrite(first)], [entryWrite(second)]], + (index, seq) => events.push(`harness ${index} ${seq}`), + context, + ); + await entered.promise; + const waiting = other.commit( + [[{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }]], + (index, seq) => events.push(`other ${index} ${seq}`), + context, + ); + await macrotask(); + expect(events).toEqual([]); + const last = store.lastSeq; + release.resolve(); + await Promise.all([series, waiting]); + expect(events).toEqual([`harness 0 ${last + 1}`, `harness 1 ${last + 2}`, `other 0 ${last + 3}`]); + expect((await outboxSeqs(store)).slice(-3)).toEqual([last + 1, last + 2, last + 3]); + await other.close(context); + await store.close(context); + }); + + it("a read during a series sees all of it or none of it", async () => { + const { store, storage, faults } = await counted(); + const [first, second] = [await storage.mintId(), await storage.mintId()]; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + // Held after the first batch's writes, in the second batch's check. + let claimReads = 0; + faults.hold = (sql) => { + if (!sql.includes("record_ids") || ++claimReads !== 2) return undefined; + entered.resolve(); + return release.promise; + }; + const series = storage.commit([[entryWrite(first)], [entryWrite(second)]], () => {}, context); + await entered.promise; + const read = Promise.all([storage.entry(first, context), storage.entry(second, context)]); + await macrotask(); + release.resolve(); + await series; + const [one, two] = await read; + expect([one === undefined, two === undefined]).toEqual([false, false]); + await store.close(context); + }); + + it("an export holds the queue: a series waits for it and is not in its archive", async () => { + const { store, storage } = await counted(); + const entry = await storage.mintId(); + const last = store.lastSeq; + const parts = store.export(context); + const manifest = await parts.next(); + expect(manifest.value).toMatchObject({ kind: "manifest", lastSeq: last }); + let settled = false; + const series = storage + .commit([[entryWrite(entry)]], () => {}, context) + .finally(() => { + settled = true; + }); + await macrotask(); + expect(settled).toBe(false); + await parts.return?.(); + expect(await series).toEqual({ stored: 1 }); + expect(store.lastSeq).toBe(last + 1); + await store.close(context); + }); +}); + +describe("GC", () => { + it("keeps nothing of a settled call: its batches are collected", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const { store, storage } = await counted(); + const sent = async (): Promise> => { + const batches = [[entryWrite(await storage.mintId())]]; + await storage.commit(batches, () => {}, context); + return new WeakRef(batches); + }; + const ref = await sent(); + for (let round = 0; round < 10 && ref.deref() !== undefined; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + expect(ref.deref()).toBeUndefined(); + await store.close(context); + }); +}); diff --git a/packages/durable/test/store-storage.test.ts b/packages/durable/test/store-storage.test.ts index 94b7dee3a..cd1ca25ce 100644 --- a/packages/durable/test/store-storage.test.ts +++ b/packages/durable/test/store-storage.test.ts @@ -32,14 +32,24 @@ import { describe, expect, it } from "vitest"; import { StorageRequestError } from "../src/errors.ts"; import { idFromNumber } from "../src/ids.ts"; import { type CompactionResult, createRegistry, Harness, LiveDoc, type Tx } from "../src/index.ts"; +import { SessionImpl } from "../src/session/session.ts"; import { StoreStorage } from "../src/storage/store/index.ts"; -import type { ConversationId, DocumentId, EntryId, Id, Seq, Storage, StorageWrite, TaskId } from "../src/types.ts"; +import { commitWrites } from "../src/testing/commit-writes.ts"; +import type { + CommitResult, + ConversationId, + DocumentId, + EntryId, + Id, + Seq, + Storage, + StorageWrite, + TaskId, +} from "../src/types.ts"; import { ROOT_CONVERSATION_ID } from "../src/types.ts"; import { openStoreStorage } from "./store-storage-support.ts"; const context = BACKGROUND_CONTEXT; -const adopt = (): void => {}; - /** A storage that closes and reopens its store file after every commit (Durable's reopen harness). */ class ReopeningStorage implements Storage { #current: Storage; @@ -51,10 +61,14 @@ class ReopeningStorage implements Storage { this.#path = path; } - async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context): Promise { - if (this.#closed) return this.#current.commit(writes, adopt, commitContext); + async commit( + batches: readonly (readonly StorageWrite[])[], + receive: (index: number, seq: Seq) => void, + commitContext: Context, + ): Promise { + if (this.#closed) return this.#current.commit(batches, receive, commitContext); try { - return await this.#current.commit(writes, adopt, commitContext); + return await this.#current.commit(batches, receive, commitContext); } finally { await this.#current.close(context); this.#current = await openStoreStorage(this.#path); @@ -127,7 +141,7 @@ describe("claims and close", () => { reason: "args", }); const other = await StoreStorage.open(store, "other", context); - await first.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(first, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); expect(await other.conversation(ROOT_CONVERSATION_ID, context)).toBeUndefined(); await first.close(context); await expect(first.conversation(ROOT_CONVERSATION_ID, context)).rejects.toThrow("StoreStorage is closed"); @@ -139,7 +153,7 @@ describe("claims and close", () => { it("waits at close for its calls underway, then releases its claims", async () => { const store = await memoryStore(); const storage = await StoreStorage.open(store, "harness", context); - const commit = storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + const commit = commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); const closing = storage.close(context); await commit; await closing; @@ -153,14 +167,14 @@ describe("the store's semantics (D4)", () => { const store = await memoryStore(); const storage = await StoreStorage.open(store, "harness", context); const root = { type: "conversation", value: { id: ROOT_CONVERSATION_ID } } as const; - await storage.commit([root], adopt, context); - await expect(storage.commit([root], adopt, context)).rejects.toThrow("ID 1 already belongs to conversation"); + await commitWrites(storage, [root], context); + await expect(commitWrites(storage, [root], context)).rejects.toThrow("ID 1 already belongs to conversation"); const entry = { id: idFromNumber(await storage.mintId()), conversationId: ROOT_CONVERSATION_ID, kind: "m", }; - await storage.commit([{ type: "entry", value: entry }], adopt, context); + await commitWrites(storage, [{ type: "entry", value: entry }], context); expect((await storage.entry(entry.id, context))?.entry).toEqual(entry); await store.close(context); }); @@ -169,12 +183,12 @@ describe("the store's semantics (D4)", () => { const store = await memoryStore(); const storage = await StoreStorage.open(store, "harness", context); const entry = { id: idFromNumber(500), conversationId: ROOT_CONVERSATION_ID, kind: "m" }; - await storage.commit( + await commitWrites( + storage, [ { type: "conversation", value: { id: ROOT_CONVERSATION_ID } }, { type: "entry", value: entry }, ], - adopt, context, ); expect(await storage.mintId()).toBe(501); @@ -198,7 +212,7 @@ async function recorded() { outbox: true, })) as ChordStore; const storage = await StoreStorage.open(store, "harness", context); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); return { store, storage }; } @@ -211,7 +225,8 @@ async function history(storage: Storage): Promise<{ readonly rewindable: Documen const seqs: Seq[] = []; const base = (n: number) => ({ kind: "base", version: 1, value: { n } }) as const; seqs.push( - await storage.commit( + await commitWrites( + storage, [ { type: "document.create", @@ -224,26 +239,25 @@ async function history(storage: Storage): Promise<{ readonly rewindable: Documen content: base(0), }, ], - adopt, context, ), ); seqs.push( - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id: rewindable, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], - adopt, context, ), ); seqs.push( - await storage.commit( + await commitWrites( + storage, [ { type: "document.change", id: latest, content: { kind: "delta", version: 1, ops: [["s", ["n"], 2]] } }, { type: "document.retire", id: latest }, { type: "document.create", record: { id: empty, kind: "e", scope: { kind: "session" } }, content: base(3) }, { type: "document.retire", id: empty }, ], - adopt, context, ), ); @@ -323,11 +337,11 @@ describe("head markers (the entry_heads table)", () => { (await openStore(sqliteBackend(await openNodeSqliteDatabase(path)), context, { outbox: true })) as ChordStore; let store = await open(); let storage = await StoreStorage.open(store, "harness", context); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); const entry = async (head?: EntryId): Promise => { const id = await storage.mintId(); const value = { id, conversationId: ROOT_CONVERSATION_ID, kind: "k", data: {}, ...(head ? { head } : {}) }; - await storage.commit([{ type: "entry", value }], adopt, context); + await commitWrites(storage, [{ type: "entry", value }], context); return id; }; const first = await entry(); @@ -372,7 +386,30 @@ describe("reads (slice 6b §1.4, §2.4)", () => { }); describe("commits (slice 6b §1.3, §2.5)", () => { - it("adopt in the attached host's batch, and count four statements for an entry append and for a delta", async () => { + it("publishes each commit of a Session inside the attached host's batch, and plainly without a host", async () => { + const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; + const storage = await StoreStorage.open(store, "harness", context); + const plain: boolean[] = []; + storage.publishing(() => plain.push(store.batch === undefined), context); + expect(plain).toEqual([true]); + const host = createHost({ store, onError: () => {} }); + const session = new SessionImpl(storage); + const running: boolean[] = []; + session.followCommits(() => { + running.push(store.batch!.running); + }); + await session.commitWith((tx) => tx.createRootConversation(), context); + await Promise.all([ + session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "a" }), context), + session.commit((tx) => tx.appendEntry(ROOT_CONVERSATION_ID, { kind: "b" }), context), + ]); + expect(running).toEqual([true, true, true]); + await session.close(context); + await host.close(context); + await store.close(context); + }); + + it("counts four statements for an entry append and for a delta, with a host attached", async () => { const telemetry = new InMemoryTelemetryContext(); const span = await telemetry.startSpan({ name: "test" }, (each) => each); const metered = withTelemetry(span, context); @@ -384,34 +421,26 @@ describe("commits (slice 6b §1.3, §2.5)", () => { const store = (await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context)) as ChordStore; const host = createHost({ store, onError: () => {} }); const storage = await StoreStorage.open(store, "harness", context); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); - let running: boolean | undefined; + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); const entry = { id: idFromNumber(await storage.mintId()), conversationId: ROOT_CONVERSATION_ID, kind: "m", }; const before = counts(); - await storage.commit( - [{ type: "entry", value: entry }], - () => { - running = store.batch!.running; - }, - metered, - ); + await storage.commit([[{ type: "entry", value: entry }]], () => {}, metered); const appended = counts() - before; - expect(running).toBe(true); const id = await storage.mintId(); const record = { id, kind: "s", scope: { kind: "session" } } as const; - await storage.commit( + await commitWrites( + storage, [{ type: "document.create", record, content: { kind: "base", version: 1, value: { n: 0 } } }], - adopt, context, ); const changed = counts(); - await storage.commit( + await commitWrites( + storage, [{ type: "document.change", id, content: { kind: "delta", version: 1, ops: [["s", ["n"], 1]] } }], - adopt, metered, ); expect([appended, counts() - changed]).toEqual([4, 4]); @@ -539,10 +568,11 @@ describe("Durable's views as Chord sources on the store (slice 6b §3)", () => { describe("review regressions", () => { it('a head-marker entry whose kind is "refused" is returned, not taken for a request error', async () => { const storage = await openStoreStorage(":memory:"); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); const first = await storage.mintId(); const marker = await storage.mintId(); - await storage.commit( + await commitWrites( + storage, [ { type: "entry", value: { id: first, conversationId: ROOT_CONVERSATION_ID, kind: "message" } }, { @@ -550,7 +580,6 @@ describe("review regressions", () => { value: { id: marker, conversationId: ROOT_CONVERSATION_ID, kind: "refused", head: first }, }, ], - adopt, context, ); expect((await storage.findLatestHeadMarker(ROOT_CONVERSATION_ID, undefined, context))?.id).toBe(marker); @@ -558,16 +587,6 @@ describe("review regressions", () => { expect(page.items.map(({ kind }) => kind)).toEqual(["refused", "message"]); await storage.close(context); }); - - it("an empty commit takes no seq", async () => { - const { store, storage } = await recorded(); - const before = store.lastSeq; - const adopted: Seq[] = []; - expect(await storage.commit([], (seq) => adopted.push(seq), context)).toBe(before); - expect(store.lastSeq).toBe(before); - expect(adopted).toEqual([before]); - await store.close(context); - }); }); describe("GC", () => { @@ -577,7 +596,7 @@ describe("GC", () => { const store = await memoryStore(); const opened = async (): Promise> => { const storage = await StoreStorage.open(store, "harness", context); - await storage.commit([{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], adopt, context); + await commitWrites(storage, [{ type: "conversation", value: { id: ROOT_CONVERSATION_ID } }], context); await storage.close(context); return new WeakRef(storage); }; diff --git a/packages/durable/test/system-order-cache-e2e.test.ts b/packages/durable/test/system-order-cache-e2e.test.ts index 0d67629d5..f6581ac4a 100644 --- a/packages/durable/test/system-order-cache-e2e.test.ts +++ b/packages/durable/test/system-order-cache-e2e.test.ts @@ -4,7 +4,6 @@ import { type Model, type Models, normalizeContext, - Type, uuidv7, } from "@earendil-works/pi-ai"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; @@ -18,6 +17,7 @@ import { hook, MemoryStorage, } from "@earendil-works/pi-durable"; +import { obj, str } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { streamSimple as streamSimpleAnthropic } from "../../ai/src/api/anthropic-messages.ts"; import { resolveApiKey } from "../../ai/test/oauth.ts"; @@ -41,7 +41,7 @@ function instructions(nonce: string): string { return lines.join("\n"); } -const Parameters = Type.Object({ text: Type.String() }); +const Parameters = obj({ text: str() }); const first = defineTool({ name: "probe_first", description: "Unused probe tool.", diff --git a/packages/durable/test/tool-args-typebox.test.ts b/packages/durable/test/tool-args-typebox.test.ts new file mode 100644 index 000000000..7a606cf96 --- /dev/null +++ b/packages/durable/test/tool-args-typebox.test.ts @@ -0,0 +1,61 @@ +// Tool arguments typed from a TypeBox-built parameter schema. Durable does not depend on TypeBox: the schema below +// has TypeBox 1.x's type shapes as its d.mts declare them; pi-schema's own tests check the shapes against TypeBox. + +import { defineTool, type ToolRegistration } from "@earendil-works/pi-durable"; +import { describe, expectTypeOf, it } from "vitest"; + +interface TString { + "~kind": "String"; + type: "string"; +} + +interface TInteger { + "~kind": "Integer"; + type: "integer"; +} + +interface TArray { + "~kind": "Array"; + type: "array"; + items: Items; +} + +interface TObject { + "~kind": "Object"; + type: "object"; + properties: Properties; + required: string[]; +} + +type TOptional = Schema & { "~optional": true }; + +const parameters: TObject<{ path: TString; offset: TOptional; tags: TArray }> = { + "~kind": "Object", + type: "object", + properties: { + path: { "~kind": "String", type: "string" }, + offset: { "~kind": "Integer", type: "integer", "~optional": true }, + tags: { "~kind": "Array", type: "array", items: { "~kind": "String", type: "string" } }, + }, + required: ["path", "tags"], +}; + +type Args = { path: string; offset?: number; tags: string[] }; + +describe("tool arguments of a TypeBox schema", () => { + it("are typed by the schema in execute() and prepareArguments()", () => { + const tool = defineTool({ + name: "read", + description: "Reads a file", + parameters, + async execute(args) { + expectTypeOf(args).toEqualTypeOf(); + return {}; + }, + }); + expectTypeOf[0]>().toEqualTypeOf(); + expectTypeOf< + ReturnType["prepareArguments"]>> + >().toEqualTypeOf(); + }); +}); diff --git a/packages/durable/test/tool-output-bench.ts b/packages/durable/test/tool-output-bench.ts index 88fec589e..ac7a54d29 100644 --- a/packages/durable/test/tool-output-bench.ts +++ b/packages/durable/test/tool-output-bench.ts @@ -7,6 +7,7 @@ * then close the Harness mid-round (persistent backends) to measure replay of the round's delta chain on reopen, and * finish the run to measure the stored size after the round's base and reclamation. Throughput scenarios push 1 GiB. */ + import { execFile } from "node:child_process"; import { mkdtemp, open, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -16,7 +17,7 @@ import { promisify } from "node:util"; import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; -import { Type } from "typebox"; +import { obj } from "@earendil-works/pi-schema"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { createRegistry, defineTool, Harness, LiveDoc, MemoryStorage, type Storage } from "../src/index.ts"; import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; @@ -145,7 +146,7 @@ async function runScenario(scenario: Scenario): Promise { defineTool({ name: "emit", description: "", - parameters: Type.Object({}), + parameters: obj({}), outputLimits: { retain: scenario.retain }, execute: async (_args, api, callContext) => { let counter = 0; @@ -261,7 +262,7 @@ async function openStorage(backend: Backend, path: string): Promise { return openNodeJsonlStorage(path, context); } -/** Measure every Storage commit. */ +/** Measure every Storage commit call. */ function timed(storage: Storage, latencies: number[]): Storage { return new Proxy(storage, { get(target, property) { diff --git a/packages/durable/test/tool-search.test.ts b/packages/durable/test/tool-search.test.ts index 895240bbb..988c85d5a 100644 --- a/packages/durable/test/tool-search.test.ts +++ b/packages/durable/test/tool-search.test.ts @@ -1,4 +1,4 @@ -import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, @@ -9,6 +9,7 @@ import { MemoryStorage, type ToolRegistration, } from "@earendil-works/pi-durable"; +import { num, obj, opt } from "@earendil-works/pi-schema"; import { describe, expect, it } from "vitest"; import { createToolSearchExtension } from "../src/tool-search/index.ts"; import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; @@ -19,7 +20,7 @@ function tool(name: string, description: string, extra: Partial ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); @@ -120,7 +121,7 @@ describe("deferred tools and tool_search", () => { const grow = defineTool({ name: "grow", description: "grow", - parameters: Type.Object({}), + parameters: obj({}), execute: async () => ({ output: [], control: { addTools: ["github_get_issue", "alpha", "ghost"] } }), }); addTool(chat.registry, grow); diff --git a/packages/durable/test/tools-read-differential.test.ts b/packages/durable/test/tools-read-differential.test.ts index e8486be10..b1e30fe4c 100644 --- a/packages/durable/test/tools-read-differential.test.ts +++ b/packages/durable/test/tools-read-differential.test.ts @@ -17,8 +17,10 @@ import { DEFAULT_MAX_BYTES, formatSize, truncateHead } from "../src/truncate.ts" function referenceRead(bytes: Uint8Array, path: string, offset: number | undefined, limit: number | undefined) { const mimeType = detectSupportedImageMimeType(bytes); if (mimeType) { + const block = { type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }; return { - output: [{ type: "image", data: Buffer.from(bytes).toString("base64"), mimeType }], + output: [block], + structuredOutput: block, diagnostics: [{ severity: "info", code: "image", message: `Read image file [${mimeType}].` }], }; } @@ -73,6 +75,7 @@ function referenceRead(bytes: Uint8Array, path: string, offset: number | undefin } return { output: outputText === "" ? [] : [{ type: "text", text: outputText }], + structuredOutput: outputText, ...(details === undefined ? {} : { details }), diagnostics, }; diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index 96bd4e91e..6c101bdd4 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + ### Breaking Changes - `RemoteWatcher` follows Durable's `FileWatcher`: `close(context)` is `stop(context)`, and its changes are tagged (`{ kind: "paths", paths }`, `{ kind: "overflow" }`, `{ kind: "failed", error }`). The daemon's `watch` protocol is unchanged. Migration: `await watcher.close(ctx)` becomes `await watcher.stop(ctx)`; test `change.kind` instead of `"overflow" in change` or `"error" in change`. diff --git a/packages/env/daemon/src/watch.rs b/packages/env/daemon/src/watch.rs index 18d202b82..b53c83d4b 100644 --- a/packages/env/daemon/src/watch.rs +++ b/packages/env/daemon/src/watch.rs @@ -4,7 +4,7 @@ use crate::errors::Failure; use crate::exec::Control; -use crate::frame::{EVENT, Frame}; +use crate::frame::{EVENT, Frame, MAX_PAYLOAD}; use crate::output::Output; use crate::sys; use notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; @@ -662,7 +662,7 @@ impl FileWatcher<'_> { if !changed.is_empty() && !self.cancelled() { let mut paths: Vec = changed.into_iter().collect(); sort_like_javascript(&mut paths); - self.send(json!({ "kind": "change", "paths": paths })); + self.send(change_event(paths)); } true } @@ -675,6 +675,16 @@ impl FileWatcher<'_> { } } +/// A `change` event listing `paths`, or an `overflow` one when the list would not fit in a frame: a frame that is too +/// large would end the connection's output. +fn change_event(paths: Vec) -> Value { + let event = json!({ "kind": "change", "paths": paths }); + if serde_json::to_vec(&event).map_or(true, |json| json.len() > MAX_PAYLOAD) { + return json!({ "kind": "change", "overflow": true }); + } + event +} + fn parse_targets(json: &Value) -> Result, Failure> { let invalid = || Failure::new("EINVAL", "watch needs targets"); json["targets"] @@ -776,3 +786,22 @@ pub fn run(id: u32, json: &Value, output: &Output, control: &Control) -> Result< } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn change_event_lists_paths_that_fit() { + let event = change_event(vec!["/a".to_string(), "/b".to_string()]); + assert_eq!(event, json!({ "kind": "change", "paths": ["/a", "/b"] })); + } + + #[test] + fn change_event_overflows_when_paths_do_not_fit_in_a_frame() { + let path = "x".repeat(4096); + let count = MAX_PAYLOAD / path.len() + 1; + let event = change_event(vec![path; count]); + assert_eq!(event, json!({ "kind": "change", "overflow": true })); + } +} diff --git a/packages/env/package.json b/packages/env/package.json index ab09e6546..f78d49c20 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-env", - "version": "1.4.0", + "version": "1.5.0", "description": "Remote execution environments for Pi Durable: a small daemon deployed over SSH and its ExecutionEnv client", "type": "module", "main": "./dist/index.js", @@ -22,7 +22,7 @@ "CHANGELOG.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "build:daemon": "cargo build --manifest-path daemon/Cargo.toml --locked", "test": "vitest --run", @@ -45,11 +45,10 @@ "node": ">=22.19.0" }, "dependencies": { - "@earendil-works/chord": "1.4.0", - "@earendil-works/pi-durable": "1.4.0" + "@earendil-works/chord": "1.5.0", + "@earendil-works/pi-durable": "1.5.0" }, "devDependencies": { - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/evals/package.json b/packages/evals/package.json index e7cc7bde7..f740c3368 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -1,10 +1,10 @@ { "name": "@earendil-works/pi-evals", - "version": "1.4.0", + "version": "1.5.0", "private": true, "type": "module", "scripts": { - "clean": "shx rm -rf .eval", + "clean": "node ../../scripts/package-files.mjs clean .eval", "eval": "npm run eval:host && npm run eval:docs --", "eval:host": "vitest run --config vitest.evals.config.ts --project host", "eval:audit": "vitest run --config vitest.evals.config.ts --project audit", @@ -12,11 +12,10 @@ "test": "vitest run --config vitest.test.config.ts" }, "devDependencies": { - "@earendil-works/pi-ai": "1.4.0", - "@earendil-works/pi-coding-agent": "1.4.0", + "@earendil-works/pi-ai": "1.5.0", + "@earendil-works/pi-coding-agent": "1.5.0", "@types/node": "22.19.19", "@vitest-evals/core": "0.15.0", - "shx": "0.3.4", "vitest": "4.1.12", "vitest-evals": "0.15.0" } diff --git a/packages/mcp/CHANGELOG.md b/packages/mcp/CHANGELOG.md index 74908a649..a0e3f2cc8 100644 --- a/packages/mcp/CHANGELOG.md +++ b/packages/mcp/CHANGELOG.md @@ -2,6 +2,12 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + +### Changed + +- README: the `AgentTool` wrapping example passes the MCP input schema as plain JSON Schema instead of through TypeBox's `Type.Unsafe`. + ### Fixed - OAuth metadata parsing and `startAuthorization()` now accept only `http:` and `https:` URLs. Before, a server could supply a `file:` or custom protocol authorization endpoint (for example `search-ms:`), which the platform URL handler would launch on sign-in. diff --git a/packages/mcp/README.md b/packages/mcp/README.md index b09e043fb..ebb1b2973 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -36,7 +36,6 @@ Wrapping an MCP tool as a `pi-agent-core` `AgentTool`: ```typescript import type { AgentTool } from "@earendil-works/pi-agent-core"; import { toLlmContent } from "@earendil-works/pi-mcp"; -import { Type } from "typebox"; const tools: AgentTool[] = (await client.listTools()).map((tool) => ({ // Providers allow at most 64 characters of [A-Za-z0-9_-]. @@ -44,7 +43,7 @@ const tools: AgentTool[] = (await client.listTools()).map((tool) => ({ label: tool.title ?? tool.name, description: tool.description ?? tool.name, // Providers require an object schema, and some reject one without `properties`. - parameters: Type.Unsafe({ ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {} }), + parameters: { ...tool.inputSchema, type: "object", properties: tool.inputSchema.properties ?? {} }, execute: async (_toolCallId, params, signal) => { const result = await client.callTool(tool.name, params as Record, { signal }); // MCP reports tool failures in the result instead of as a protocol error. diff --git a/packages/mcp/package.json b/packages/mcp/package.json index b016e8067..8a9bac663 100644 --- a/packages/mcp/package.json +++ b/packages/mcp/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-mcp", - "version": "1.4.0", + "version": "1.5.0", "description": "Standalone Model Context Protocol client for pi and other applications", "type": "module", "main": "./dist/index.js", @@ -30,7 +30,7 @@ "LICENSES" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" @@ -55,7 +55,6 @@ }, "devDependencies": { "@types/cross-spawn": "6.0.6", - "shx": "0.3.4", "vitest": "4.1.12" } } diff --git a/packages/schema/CHANGELOG.md b/packages/schema/CHANGELOG.md new file mode 100644 index 000000000..1fb3218b0 --- /dev/null +++ b/packages/schema/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## [Unreleased] + +## [1.5.0] - 2026-11-02 + +### Added + +- Initial release: `compile(schema, options)` validates JSON Schema drafts 2020-12 and 7 without `eval` or `new Function`, compiling a schema once into plain closures. `Validator.check` is the boolean check (a type guard for schemas built with the builders), `Validator.errors` reports each failure with a JSON Pointer path. Format checks are in the optional `@earendil-works/pi-schema/formats` entry point. +- Schema builders (`str`, `num`, `int`, `bool`, `nul`, `unknown`, `lit`, `oneOf`, `obj`, `opt`, `arr`, `tuple`, `map`, `union`, `nullable`, `bytes`) that return plain JSON Schema typed with the values it accepts (`Static`). +- `compile(schema, { coerce: true })` adds `coerce(value)` and `parse(value)` for loosely typed input such as a model's tool call arguments: optional `null`s dropped, TypeBox 1.3.27's `Value.Convert` rules keyed on JSON Schema keywords (numeric strings, `"true"`, single values to arrays, `const`/`enum` members, union branches), then Ajv-style primitive coercion by `type`. `coerce` copies only the paths it changes and returns the input itself when nothing changes; `parse` coerces, then checks. Validators compiled without the option have neither; the README lists every rule. +- `Static` also types TypeBox-built schemas, from TypeBox 1.x's type shapes and without importing TypeBox: String, Number, Integer, Boolean, Null, Literal, Array, Object (with `Type.Optional` and `Type.Readonly` properties), Union, Record, Unsafe, Unknown and Any give TypeBox's static type; other TypeBox kinds give `unknown`. +- `unresolvedRefs: "reject"` compiles a `$ref` that names nothing as `false` (the error is `unresolved reference `) instead of throwing, for schemas whose references may dangle. +- `formats` also checks `url` and `json-pointer-uri-fragment`, as TypeBox and ajv-formats do. +- `npm run bench` compares conformance, speed (with and without eval), setup cost, memory and bundle size with ata-validator (generated code and its interpreter), Ajv, Zod (native schemas, and `z.fromJSONSchema` for the test suite), TypeBox, @cfworker/json-schema and omptype. diff --git a/packages/schema/LICENSES/typebox-LICENSE b/packages/schema/LICENSES/typebox-LICENSE new file mode 100644 index 000000000..0330da432 --- /dev/null +++ b/packages/schema/LICENSES/typebox-LICENSE @@ -0,0 +1,23 @@ +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file diff --git a/packages/schema/README.md b/packages/schema/README.md new file mode 100644 index 000000000..f1ff34f4d --- /dev/null +++ b/packages/schema/README.md @@ -0,0 +1,296 @@ +# @earendil-works/pi-schema + +JSON Schema builders and validation without `eval`. A schema compiles once into plain closures, so it checks values +at close to the speed of validators that generate code, also under a strict Content Security Policy, and needs no +dependencies. + +- Drafts 2020-12 and 7. Passes 2,222 of the 2,223 core tests of the official + [JSON-Schema-Test-Suite](https://github.com/json-schema-org/JSON-Schema-Test-Suite). +- Builders return plain JSON Schema typed with the values it accepts; a compiled check narrows to that type. +- Errors as JSON Pointer paths. A missing or unknown field is reported at the field. +- Never modifies a schema; frozen schemas work. +- 8.5 KB minified and gzipped. Format checks are an optional import. + +## Installation + +```bash +npm install @earendil-works/pi-schema +``` + +## Usage + +```typescript +import { compile, int, obj, oneOf, opt, type Static, str } from "@earendil-works/pi-schema"; + +const Read = obj({ + path: str({ description: "Path to the file", minLength: 1 }), + offset: opt(int({ minimum: 1 })), + mode: oneOf(["text", "binary"]), +}); +type Read = Static; // { path: string; mode: "text" | "binary"; offset?: number } + +const read = compile(Read); +if (read.check(value)) value.path; // value: Read + +read.errors({ offset: 0, extra: true }); +// [ +// { path: "/path", keyword: "required", message: "missing" }, +// { path: "/mode", keyword: "required", message: "missing" }, +// { path: "/offset", keyword: "minimum", message: "expected >= 1" }, +// { path: "/extra", keyword: "additionalProperties", message: "unknown field" }, +// ] +``` + +`Read` is plain JSON Schema (`{ type: "object", properties: {...}, required: ["path", "mode"], additionalProperties: +false }`), ready for a tool definition or `JSON.stringify`. `compile` takes any JSON Schema, built with the builders +or not. + +`check(value)` is the fast path: a boolean, no allocations for most schemas, and a plain function you can pass +around. `errors(value)` explains a failure and returns `[]` for a valid value. It is built on the first failure and +kept, so a schema whose values always pass never pays for it. + +### Builders + +| builder | schema | type | +|---|---|---| +| `str({ minLength, maxLength, pattern, format })` | `{ type: "string", ... }` | `string` | +| `num({ minimum, maximum, exclusiveMinimum, exclusiveMaximum, multipleOf })`, `int(...)` | `{ type: "number" }`, `{ type: "integer" }` | `number` | +| `bool()`, `nul()`, `unknown()` | `{ type: "boolean" }`, `{ type: "null" }`, `{}` | `boolean`, `null`, `unknown` | +| `lit(value)` | `{ const: value }` | the literal | +| `oneOf(["a", "b"])` | `{ enum: [...] }` | `"a" \| "b"` | +| `obj({ a: str(), b: opt(num()) }, { open })` | closed object; `open: true` allows other properties | `{ a: string; b?: number }` | +| `arr(items, { minItems, maxItems, uniqueItems })` | `{ type: "array", items }` | `T[]` | +| `tuple([str(), num()])` | `prefixItems`, no more items | `[string, number]` | +| `map(value)` | `{ type: "object", additionalProperties: value }` | `Record` | +| `union([...])` | `{ anyOf: [...] }` | the union | +| `nullable(schema)` | `{ anyOf: [schema, { type: "null" }] }` | `T \| null` | +| `bytes({ minLength, maxLength })` | `{ type: "bytes", ... }` | `Uint8Array` | + +Every builder also takes `title`, `description`, `examples`, `deprecated` and `default`. A `default` is an annotation; +validation never inserts it. + +A union of objects tagged by a `lit()` property checks and reports by the tag: only the branch the tag selects runs. + +```typescript +const Block = union([obj({ kind: lit("text"), text: str() }), obj({ kind: lit("image"), width: int({ minimum: 0 }) })]); +compile(Block).errors({ kind: "video" }); +// [{ path: "/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }] +``` + +`Static` also types schemas built with TypeBox 1.x, read from TypeBox's type shapes (pi-schema does not import +TypeBox): `String`, `Number`, `Integer`, `Boolean`, `Null`, `Literal`, `Array`, `Object` (with `Type.Optional` and +`Type.Readonly` properties), `Union`, `Record`, `Unsafe`, `Unknown` and `Any` give TypeBox's own static type, so +`Static` is `{ path: string }`. Other TypeBox kinds (`Enum`, `Tuple`, +`Intersect`, `Cyclic`, ...) give `unknown` where they stand. Other JSON Schema objects give `unknown`. + +### Options + +```typescript +import { formats } from "@earendil-works/pi-schema/formats"; + +const validator = compile(schema, { + draft: "7", // "2020-12" (default) or "7" + formats, // check `format`; without it `format` is an annotation, as 2020-12 specifies + schemas: { "https://example.com/point.json": point }, // documents for $refs to other documents, by URI + unresolvedRefs: "reject", // a $ref that names nothing rejects values at its place; default "throw" + coerce: true, // adds coerce() and parse(), see Coercion +}); +``` + +`formats` has every format 2020-12 defines (`date-time`, `email`, `uri`, `uuid`, `ipv6`, ...), plus `url` and +`json-pointer-uri-fragment`, which TypeBox and ajv-formats also check. Pass your own map to check fewer formats, or +custom ones. + +A `$ref` that names nothing (a missing definition, a document not in `schemas`) throws when the schema compiles. With +`unresolvedRefs: "reject"` it compiles as `false` instead: only values at that place fail, with the error +`unresolved reference `, and the rest of the schema works. Use it for schemas from other systems, such as MCP +servers, whose references may dangle. + +### TypeBox schemas + +TypeBox 1.x schemas are JSON Schema, so they compile as they are: + +```typescript +import Type from "typebox"; + +const user = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.String()) })); +``` + +TypeBox's JavaScript-only types (`BigInt`, `Undefined`, `Function`, ...), refinements and codecs are not JSON Schema +and are not supported. + +### Coercion + +`compile(schema, { coerce: true })` adds `coerce()` and `parse()`, for loosely typed input such as a model's tool call +arguments. Without the option the validator has neither, and its `check` pays nothing for them: the coercion rules are +built on the first `coerce()` or `parse()`. + +```typescript +const read = compile(Read, { coerce: true }); + +read.coerce({ path: "a.txt", offset: "10", mode: "text" }); // { path: "a.txt", offset: 10, mode: "text" } + +const result = read.parse({ path: "a.txt", offset: "ten", mode: "text" }); +if (result.ok) result.value.offset; // number +else result.errors; // [{ path: "/offset", keyword: "type", message: "expected integer, got string" }] +``` + +`coerce(value)` returns the coerced value. It copies only the paths it changes, returns `value` itself when nothing +changes, and never modifies `value`. A value no rule applies to stays as it is. `parse(value)` coerces, then checks: +`{ ok: true, value }` with the coerced value, or `{ ok: false, errors }` with the errors of the coerced value. + +Three passes run in order: + +1. **Optional nulls.** A `null` in an optional property whose schema rejects `null` is dropped, as if omitted. +2. **Conversions**, ported from TypeBox 1.3.27's `Value.Convert` and keyed on JSON Schema keywords (the first that + applies to a schema): `$ref` follows the reference; `const` and `enum` take the member the value converts to; + `anyOf`/`oneOf` keep a value one branch accepts, else take the first branch's conversion the whole schema + accepts; `allOf` converts with each member in turn; then by `type`: + - `string`: numbers and booleans to their text, `null` to `"null"`. + - `number`: numeric strings (`""` is 0), `"true"`/`"false"`, booleans to 1/0, `null` to 0. + - `integer`: as `number`, then truncated (`"3.5"` and `3.5` become 3). + - `boolean`: `"true"`/`"false"` (any case), `"1"`/`"0"`, 1/0, `null` to `false`. + - `null`: `"null"`, `"undefined"`, `""`, `"0"`, 0 and `false`. + - `array`: a single value becomes a one-item array, then each item converts; a tuple (`prefixItems` or an `items` + array) converts its items and never wraps. + - `object`: each property converts by `properties`, `patternProperties` or `additionalProperties`. +3. **Primitive coercion by type**, as Ajv's `coerceTypes`, for what pass 2 leaves (type arrays such as + `["integer", "null"]`): numbers from numeric strings and booleans, booleans from `"true"`/`"false"` and 1/0, + strings from numbers and booleans, `null` from `""`, 0 and `false`, and `null` to `""`, 0 or `false`. A value that + already has one of a type array's types stays. + +Where passes 2 and 3 disagree, pass 2 wins: `null` for a required string becomes `"null"`, not `""`. A union's +branches convert in order and only until one is accepted; a union node that also declares properties converts them into +each branch's candidate before checking it. A recursive schema (a tree through `$ref`) that meets a value it cannot +convert leaves the value as it is, so `check` reports it, and results through a `$ref` are reused within one call, so a +recursive union does not convert (or explain) the same child once per branch. + +Pass 1 drops optional nulls inside the values of `properties`, `patternProperties` and `additionalProperties` alike; a +pattern or additional key itself is never dropped. Items past a draft-7 `items` array (`additionalItems`) are not +coerced by any pass. + +### Values + +- A property whose value is `undefined` counts as absent, as in TypeBox. +- `{ type: "bytes", minLength, maxLength }` matches a `Uint8Array` of that length, an extension for values that never + pass through JSON. + +## Supported keywords + +- Every validation keyword of drafts 2020-12 and 7, including draft 7's `dependencies` and `items` arrays with + `additionalItems`. +- `$id`, `$anchor`, `$ref` (local, by anchor, relative and remote), `$dynamicRef` and `$dynamicAnchor`. +- `unevaluatedProperties` and `unevaluatedItems`. +- `pattern` and `patternProperties` are ECMAScript regular expressions with the `u` flag. +- Not supported: meta-schemas that switch vocabularies on or off (`$vocabulary`), `contentMediaType` and + `contentEncoding` assertions (optional in the spec), and draft 2019-09's `$recursiveRef`. + +## Benchmarks + +Node 24.4.1 on one core of an i9-13900. The other libraries: + +- ata-validator 1.61.3: its generated code, and its eval-free interpreter chosen with `engine: "interpreter"`; +- Ajv 8.20.0, with ajv-formats 3.0.1 for the format tests; +- Zod 4.6.7: native Zod schemas for speed, setup and memory, and `z.fromJSONSchema` for the test suite; +- TypeBox 1.3.27; +- @cfworker/json-schema 4.1.1; +- @oh-my-pi/omptype 18.13.3, through `fromJsonSchema`. + +| test suite (tests passed) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| draft 2020-12 core (1,298) | 1,297 | 1,298 | 1,298 | 1,236 | 693 | 1,276 | 1,249 | 863 | +| draft 7 core (925) | 925 | 925 | 925 | 917 | 632 | 924 | 918 | 659 | +| draft 2020-12 format (895), with format checks | 895 | 814 | 814 | 750 | 412 | 895 | 740 | 412 | + +| check, valid value | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 410 ns | 180 ns | 836 ns | 627 ns | 1.3 µs | 475 ns | 43 µs | 730 ns | +| union of 5 messages | 86 ns | 37 ns | 212 ns | 70 ns | 297 ns | 95 ns | 11 µs | 96 ns | +| tool schema (read) | 31 ns | 10 ns | 41 ns | 13 ns | 41 ns | 9 ns | 2.0 µs | 9 ns | +| MCP tool schema with `$ref` | 161 ns | 64 ns | 314 ns | 136 ns | 927 ns | 87 ns | 15 µs | 1.9 µs | +| recursive `$ref` schema | 755 ns | 307 ns | 1.9 µs | 378 ns | 2.7 µs | 1.2 µs | 41 µs | 7.5 µs | + +| same, without eval (strict CSP) | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| nested object with tagged unions | 411 ns | 1.6 µs | 844 ns | fails | 3.4 µs | 44 µs | 44 µs | 2.7 µs | +| union of 5 messages | 87 ns | 396 ns | 209 ns | fails | 652 ns | 10 µs | 11 µs | 501 ns | +| MCP tool schema with `$ref` | 164 ns | 484 ns | 326 ns | fails | 2.0 µs | 39 µs | 13 µs | 2.2 µs | +| recursive `$ref` schema | 746 ns | 2.8 µs | 1.9 µs | fails | 7.3 µs | 130 µs | 42 µs | 9.0 µs | + +| cost | pi-schema | ata | ata interpreter | Ajv | Zod | TypeBox | cfworker | omptype | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| compile a new schema | 20 µs | 4.9 µs | 1.3 µs | 1.4 ms | 151 µs | 210 µs | 22 µs | 28 µs | +| its first check | 0.8 µs | 335 µs | 87 µs | 195 µs | 466 µs | 41 µs | 7.1 µs | 109 µs | +| memory, 3,200 distinct schemas | 7.1 MB | 36.0 MB | 51.3 MB | 59.7 MB | 170.8 MB | 54.7 MB | 7.6 MB | 19.9 MB | +| bundle, minified + gzip | 8.5 KB | 134.0 KB | 134.0 KB | 39.7 KB | 24.4 KB | 35.2 KB | 5.8 KB | 38.8 KB | + +Where code generation is unavailable: + +- **ata** answers through its interpreted engine without being told. That automatic fallback measured slower than the + engine chosen with `engine: "interpreter"`. +- **Ajv** cannot compile at all: `compile()` throws `EvalError`. +- **Zod** notices and parses objects without its generated code. +- **TypeBox** falls back to its interpreter. +- **omptype** throws `EvalError` once per schema, then validates through its interpreter. + +ata, Zod and omptype compile on a schema's first check, so their compile cost shows there. ata's bundle is for the +browser without code splitting; with splitting, its first load is about 53 KB plus a 76 KB generator chunk loaded +later. + +Zod is not a JSON Schema validator: + +- **Speed, setup and memory:** Zod gets the same schemas written as native Zod schemas (`z.strictObject`, + `z.discriminatedUnion`, `z.lazy`), with `.describe()` for the corpus descriptions, and `safeParse` as the check. + Its memory includes building the schemas, which the other libraries receive as JSON. Without the descriptions it + is 131 MB. +- **Test suite:** through `z.fromJSONSchema`, which is semi-experimental. It refuses `unevaluatedProperties`, + `unevaluatedItems`, `dependentSchemas`, `dependentRequired`, `if`/`then`/`else`, `not`, and every `$ref` that is + not local. A refused schema fails all its tests: 405 of the 1,298 draft 2020-12 core tests and 107 of the 925 draft + 7 core tests. +- **Bundle:** measured with Zod's documented `import * as z from "zod"`. A named `import { z }` keeps every locale + (about 90 KB gzipped), and `z.fromJSONSchema` brings it to 40.0 KB; `zod/mini` is about 6 KB. + +Flat objects of simple types check in 13–80 ns, against 7–15 ns for TypeBox. Without code generation, reading a +property whose name comes from a list costs a few nanoseconds. + +### Reproduce + +`npm run bench` in `packages/schema` takes about 12 minutes. It runs every library through its fastest boolean path +(ata's `isValidObject()`) and keeps the object each returns. ata runs with `ATA_NO_NATIVE=1`: its optional native +addon only serves its buffer APIs. The scripts are in `test/bench/`. + +- **Conformance:** every library on the vendored test suite, with the remote documents and meta-schemas the tests + reference (Zod through `z.fromJSONSchema`, which takes no remote documents). ata reads the draft only from `$schema`, so draft 7 schemas without one get it declared. ata's defaults + (`useDefaults`, `assertFormat`) are turned off to match the other libraries; formats are checked only in the + format tests, by each library's own checks. +- **Speed:** realistic schemas, in ns per check after warm-up. Each result is the median of 15 rounds, with the + libraries' order rotated in each round. +- **Speed without eval:** the same, run with `--disallow-code-generation-from-strings`. +- **Setup:** compiling 1,000 schemas a library has not seen, and their first checks. +- **Memory:** 3,200 distinct tool-like schemas, compiled and each checked 3 times, each library in its own process. +- **Size:** esbuild browser bundles of what a validator needs, minified and gzipped. + +### Code generated at build time + +Ajv (`ajv/dist/standalone`) and ata (`ata build`, `ata compile`, `toStandaloneModule()`) can also generate their +validation code when the app is built. The app then imports plain modules and evaluates nothing at run time, so those +checks run at code-generation speed under a strict CSP. This is not in the benchmark, because it is a different +deal: + +- **A build step per schema.** Every schema must be known when the app is built. Schemas that arrive at run time + (MCP servers, extensions, user input) cannot be validated this way. +- **Code per schema.** In a separate measurement on the same 3,200 schemas: + - Ajv: 1.8 ms to build and 8.9 KB of JavaScript per schema; + - ata: 1.1 ms and 13 KB per schema, or 0.3–0.8 KB gzipped when only its `isValid` export is bundled; + - importing all 3,200 generated validators took 40 MB of heap (Ajv, one module) and 68 MB (ata, one module per + schema). + +pi-schema compiles at run time in 25 µs per schema, generates no code, and covers schemas known at build time and +schemas that arrive later. + +## Development + +`npm test` runs the official test suite (drafts 2020-12 and 7, including the optional and format tests, vendored in +`test/fixtures/`) and the package's own tests. + +The format checks are adapted from TypeBox 1.3.27 (MIT); its license is in `LICENSES/`. diff --git a/packages/schema/package.json b/packages/schema/package.json new file mode 100644 index 000000000..4e176a57c --- /dev/null +++ b/packages/schema/package.json @@ -0,0 +1,60 @@ +{ + "name": "@earendil-works/pi-schema", + "version": "1.5.0", + "description": "JSON Schema validation without eval: drafts 2020-12 and 7, compiled into plain closures", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./formats": { + "source": "./src/formats/index.ts", + "types": "./dist/formats/index.d.ts", + "import": "./dist/formats/index.js" + } + }, + "sideEffects": false, + "files": [ + "dist", + "README.md", + "CHANGELOG.md", + "LICENSES" + ], + "scripts": { + "clean": "node ../../scripts/package-files.mjs clean dist", + "build": "tsc -p tsconfig.build.json", + "test": "vitest --run", + "bench": "node --expose-gc test/bench/all.ts", + "prepublishOnly": "npm run clean && npm run build" + }, + "keywords": [ + "json-schema", + "validation", + "validator", + "csp" + ], + "author": "Earendil Works", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/earendil-works/pi.git", + "directory": "packages/schema" + }, + "engines": { + "node": ">=22.19.0" + }, + "devDependencies": { + "@cfworker/json-schema": "4.1.1", + "@oh-my-pi/omptype": "18.13.3", + "ajv": "8.20.0", + "ajv-formats": "3.0.1", + "ata-validator": "1.61.3", + "typebox": "1.3.27", + "vitest": "4.1.12", + "zod": "4.6.7" + } +} diff --git a/packages/schema/src/annotations.ts b/packages/schema/src/annotations.ts new file mode 100644 index 000000000..5bc9cc4f5 --- /dev/null +++ b/packages/schema/src/annotations.ts @@ -0,0 +1,208 @@ +// Annotation tracking for `unevaluatedProperties` and `unevaluatedItems`: annotating checks record which +// properties and items a schema evaluated. Only schemas that use those keywords compile into these closures. + +import { isObject } from "./checks.ts"; +import type { Check } from "./types.ts"; + +/** What a schema evaluated: property names (or all of them), the evaluated item prefix, items matched by `contains`. */ +export interface Annotations { + props: Set | null; + allProps: boolean; + items: number; + allItems: boolean; + indexes: Set | null; +} + +/** A check that records what it evaluated into `a` when it passes. */ +export type AnnotatingCheck = (value: unknown, a: Annotations) => boolean; + +/** Empty annotations. */ +export function newAnnotations(): Annotations { + return { props: null, allProps: false, items: 0, allItems: false, indexes: null }; +} + +function merge(a: Annotations, b: Annotations): void { + if (b.props !== null) for (const k of b.props) addProp(a, k); + a.allProps ||= b.allProps; + if (b.items > a.items) a.items = b.items; + a.allItems ||= b.allItems; + if (b.indexes !== null) for (const i of b.indexes) addIndex(a, i); +} + +function addProp(a: Annotations, key: string): void { + if (a.props === null) a.props = new Set(); + a.props.add(key); +} + +function addIndex(a: Annotations, index: number): void { + if (a.indexes === null) a.indexes = new Set(); + a.indexes.add(index); +} + +export const ANNOTATING_TRUE: AnnotatingCheck = () => true; +export const ANNOTATING_FALSE: AnnotatingCheck = () => false; + +/** The indirection a recursive reference goes through. */ +export function indirectAnnotating(slot: { check: AnnotatingCheck | undefined }): AnnotatingCheck { + return (v, a) => (slot.check as AnnotatingCheck)(v, a); +} + +/** A plain check in annotating position: it evaluates nothing that unevaluated* sees. */ +export function ignoreAnnotations(check: Check): AnnotatingCheck { + return (v) => check(v); +} + +/** An annotating check run as a plain check, with fresh annotations. */ +export function withFreshAnnotations(check: AnnotatingCheck): Check { + return (v) => check(v, newAnnotations()); +} + +/** `check` with `uri` pushed onto the dynamic scope while it runs. */ +export function scopedAnnotating(scope: string[], uri: string, check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => { + scope.push(uri); + const ok = check(v, a); + scope.pop(); + return ok; + }; +} + +/** unevaluated* sees only the annotations of its own schema, never those of its siblings in the parent. */ +export function isolateAnnotations(check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => { + const b = newAnnotations(); + if (!check(v, b)) return false; + merge(a, b); + return true; + }; +} + +/** The non-annotating keywords, then each annotating part in order. */ +export function annotateAll(plain: Check, parts: readonly AnnotatingCheck[]): AnnotatingCheck { + return (v, a) => { + if (!plain(v)) return false; + for (let i = 0; i < parts.length; i++) if (!parts[i](v, a)) return false; + return true; + }; +} + +/** `properties`, `patternProperties` and `additionalProperties`. */ +export function annotateObject( + props: readonly (readonly [string, Check])[], + patterns: readonly (readonly [RegExp, Check])[], + additional: Check | undefined, +): AnnotatingCheck { + const declared = new Set(props.map(([k]) => k)); + return (v, a) => { + if (!isObject(v)) return true; + const seen: string[] = []; + for (const [k, check] of props) { + if (!Object.hasOwn(v, k)) continue; + if (!check(v[k])) return false; + seen.push(k); + } + for (const k of Object.keys(v)) { + let matched = declared.has(k); + for (const [re, check] of patterns) { + if (re.test(k)) { + if (!check(v[k])) return false; + matched = true; + seen.push(k); + } + } + if (!matched && additional !== undefined) { + if (!additional(v[k])) return false; + seen.push(k); + } + } + for (const k of seen) addProp(a, k); + return true; + }; +} + +/** `prefixItems`/`items`/`additionalItems` and `contains` with its bounds. */ +export function annotateArray( + prefix: readonly Check[], + rest: Check | undefined, + contains: Check | undefined, + minContains: number, + maxContains: number, +): AnnotatingCheck { + return (v, a) => { + if (!Array.isArray(v)) return true; + const m = Math.min(v.length, prefix.length); + for (let i = 0; i < m; i++) if (!prefix[i](v[i])) return false; + if (rest !== undefined) { + for (let i = prefix.length; i < v.length; i++) if (!rest(v[i])) return false; + a.allItems = true; + } + if (m > a.items) a.items = m; + if (contains !== undefined) { + let n = 0; + for (let i = 0; i < v.length; i++) { + if (contains(v[i])) { + n++; + addIndex(a, i); + } + } + if (n < minContains || n > maxContains) return false; + } + return true; + }; +} + +/** `anyOf` (or `oneOf` when `exactlyOne`): every passing branch contributes its annotations. */ +export function annotateAny(branches: readonly AnnotatingCheck[], exactlyOne: boolean): AnnotatingCheck { + return (v, a) => { + let n = 0; + for (const branch of branches) { + const b = newAnnotations(); + if (branch(v, b)) { + n++; + merge(a, b); + } + } + return exactlyOne ? n === 1 : n > 0; + }; +} + +/** `if`, `then` and `else`: a passing `if` contributes its annotations even without `then`. */ +export function annotateIf( + cond: AnnotatingCheck, + then: AnnotatingCheck | undefined, + otherwise: AnnotatingCheck | undefined, +): AnnotatingCheck { + return (v, a) => { + const b = newAnnotations(); + if (cond(v, b)) { + merge(a, b); + return then === undefined || then(v, a); + } + return otherwise === undefined || otherwise(v, a); + }; +} + +/** One `dependentSchemas` entry. */ +export function annotateDependent(key: string, check: AnnotatingCheck): AnnotatingCheck { + return (v, a) => !isObject(v) || !Object.hasOwn(v, key) || check(v, a); +} + +/** `unevaluatedProperties`: every property no other keyword evaluated passes `check`. */ +export function unevaluatedProperties(check: Check): AnnotatingCheck { + return (v, a) => { + if (!isObject(v) || a.allProps) return true; + for (const k of Object.keys(v)) if (!(a.props?.has(k) ?? false) && !check(v[k])) return false; + a.allProps = true; + return true; + }; +} + +/** `unevaluatedItems`: every item no other keyword evaluated passes `check`. */ +export function unevaluatedItems(check: Check): AnnotatingCheck { + return (v, a) => { + if (!Array.isArray(v) || a.allItems) return true; + for (let i = a.items; i < v.length; i++) if (!(a.indexes?.has(i) ?? false) && !check(v[i])) return false; + a.allItems = true; + return true; + }; +} diff --git a/packages/schema/src/builders.ts b/packages/schema/src/builders.ts new file mode 100644 index 000000000..2dc61a520 --- /dev/null +++ b/packages/schema/src/builders.ts @@ -0,0 +1,200 @@ +// Schema builders: short functions that return plain JSON Schema objects and carry the static type of the values +// they accept, so `Static` is the TypeScript type a valid value has. The objects are ordinary data: +// they serialize with JSON.stringify and compile with compile() like any other schema. + +import type { TypeBoxStatic } from "./typebox-static.ts"; + +declare const STATIC: unique symbol; +declare const OPTIONAL_TYPE: unique symbol; + +/** A JSON Schema object whose valid values have type `T` (a type-level brand; nothing extra exists at run time). */ +export type Schema = { readonly [keyword: string]: unknown } & { readonly [STATIC]?: T }; + +// TypeBox is tested first and in its own branch: a builder brand's fallback through TypeBoxStatic would leave +// `Static` of a generic `S` (constrained to builder schemas) unresolvable to its constraint's types. +/** + * The type of the values `S` accepts: the type a builder recorded; for a TypeBox schema (`'~kind'` or `'~unsafe'`), + * TypeBox's static type for the common kinds (see `TypeBoxStatic`); else `unknown`. + */ +export type Static = S extends { readonly "~kind": unknown } | { readonly "~unsafe": unknown } + ? TypeBoxStatic + : S extends { readonly [STATIC]?: infer T } + ? unknown extends T + ? unknown + : T + : unknown; + +/** Annotations every builder accepts: they describe the schema and never change what it accepts. */ +export interface Annotations { + readonly title?: string; + readonly description?: string; + readonly examples?: readonly T[]; + readonly deprecated?: boolean; + /** A default value, an annotation for tools and forms; validation never inserts it. */ + readonly default?: T; +} + +/** Length bounds of a string, an array or bytes. */ +export interface LengthBounds { + readonly minLength?: number; + readonly maxLength?: number; +} + +/** A string, with optional length bounds (in code points), pattern and format. */ +export function str( + options: Annotations & LengthBounds & { readonly pattern?: string; readonly format?: string } = {}, +): Schema { + return { type: "string", ...options }; +} + +/** Bounds of a number. */ +export interface NumberBounds { + readonly minimum?: number; + readonly maximum?: number; + readonly exclusiveMinimum?: number; + readonly exclusiveMaximum?: number; + readonly multipleOf?: number; +} + +/** A number within optional bounds. */ +export function num(options: Annotations & NumberBounds = {}): Schema { + return { type: "number", ...options }; +} + +/** An integer within optional bounds. */ +export function int(options: Annotations & NumberBounds = {}): Schema { + return { type: "integer", ...options }; +} + +/** A boolean. */ +export function bool(options: Annotations = {}): Schema { + return { type: "boolean", ...options }; +} + +/** `null`. */ +export function nul(options: Annotations = {}): Schema { + return { type: "null", ...options }; +} + +/** Any value. */ +export function unknown(options: Annotations = {}): Schema { + return { ...options }; +} + +/** Exactly `value` (a string, number, boolean or null). */ +export function lit( + value: V, + options: Annotations = {}, +): Schema { + return { const: value, ...options }; +} + +/** One of the given values. */ +export function oneOf( + values: V, + options: Annotations = {}, +): Schema { + return { enum: values, ...options }; +} + +/** + * Any of the given schemas. A union of objects tagged by a `lit()` property (`{ kind: lit("text"), ... }`) is checked + * by the tag: only the branch it selects runs, and its errors are the ones reported. + */ +export function union( + schemas: S, + options: Annotations> = {}, +): Schema> { + return { anyOf: schemas, ...options }; +} + +/** `schema` or `null`. */ +export function nullable(schema: Schema, options: Annotations = {}): Schema { + return { anyOf: [schema, { type: "null" }], ...options }; +} + +/** An array of `items`, with optional length bounds and unique items. */ +export function arr( + items: Schema, + options: Annotations & { + readonly minItems?: number; + readonly maxItems?: number; + readonly uniqueItems?: boolean; + } = {}, +): Schema { + return { type: "array", items, ...options }; +} + +/** A fixed-length array whose items match `items` in order. */ +export function tuple( + items: S, + options: Annotations<{ -readonly [K in keyof S]: Static }> = {}, +): Schema<{ -readonly [K in keyof S]: Static }> { + return { type: "array", prefixItems: items, items: false, minItems: items.length, ...options }; +} + +/** An object with string keys and values of `value` (a record). */ +export function map(value: Schema, options: Annotations> = {}): Schema> { + return { type: "object", additionalProperties: value, ...options }; +} + +/** A Uint8Array with optional length bounds (an extension of JSON Schema for values that never pass through JSON). */ +export function bytes(options: Annotations & LengthBounds = {}): Schema { + return { type: "bytes", ...options }; +} + +/** An optional property of obj(): `{ name: opt(str()) }` may omit `name` or set it to undefined. */ +export class Optional { + declare readonly [OPTIONAL_TYPE]?: S; + readonly schema: S; + + constructor(schema: S) { + this.schema = schema; + } +} + +/** Marks an obj() property as optional. */ +export function opt(schema: S): Optional { + return new Optional(schema); +} + +/** The properties of obj(): a schema or an opt() schema per name. */ +export type Properties = { readonly [name: string]: Schema | Optional }; + +/** The value type of an obj() schema. */ +export type StaticObject

= Simplify< + { -readonly [K in keyof P as P[K] extends Optional ? never : K]: Static } & { + -readonly [K in keyof P as P[K] extends Optional ? K : never]?: P[K] extends Optional + ? Static + : never; + } +>; + +type Simplify = { [K in keyof T]: T[K] } & {}; + +/** + * A closed object: the given properties (required unless opt()) and no others. `open: true` allows other + * properties, as tool schemas from other systems often do. + */ +export function obj( + properties: P, + options: Annotations> & { readonly open?: boolean } = {}, +): Schema> { + const { open, ...annotations } = options; + const props: Record = {}; + const required: string[] = []; + for (const [name, property] of Object.entries(properties)) { + if (property instanceof Optional) props[name] = property.schema; + else { + props[name] = property; + required.push(name); + } + } + return { + type: "object", + properties: props, + ...(required.length > 0 ? { required } : {}), + ...(open === true ? {} : { additionalProperties: false }), + ...annotations, + }; +} diff --git a/packages/schema/src/checks.ts b/packages/schema/src/checks.ts new file mode 100644 index 000000000..222d3dbde --- /dev/null +++ b/packages/schema/src/checks.ts @@ -0,0 +1,451 @@ +// Check factories: each returns a closure that captures only its arguments, never the compiler, so a compiled +// schema keeps no compile-time state alive. Leaf checks are module constants that object and array loops inline. + +import type { Check } from "./types.ts"; + +export const ALWAYS: Check = () => true; +export const NEVER: Check = () => false; +const IS_STRING: Check = (v) => typeof v === "string"; +const IS_NUMBER: Check = (v) => typeof v === "number"; +const IS_INTEGER: Check = (v) => Number.isInteger(v); +const IS_BOOLEAN: Check = (v) => typeof v === "boolean"; +const IS_NULL: Check = (v) => v === null; +export const IS_ARRAY: Check = (v) => Array.isArray(v); +const IS_BYTES: Check = (v) => v instanceof Uint8Array; +export const IS_OBJECT: Check = (v) => isObject(v); + +/** Whether `v` is a JSON object: not null, an array or bytes. */ +export function isObject(v: unknown): v is Record { + return typeof v === "object" && v !== null && !Array.isArray(v) && !(v instanceof Uint8Array); +} + +/** The check of a `type` name; `bytes` (a Uint8Array) extends JSON Schema for values that never pass through JSON. */ +export const TYPE_CHECKS: ReadonlyMap = new Map([ + ["string", IS_STRING], + ["number", IS_NUMBER], + ["integer", IS_INTEGER], + ["boolean", IS_BOOLEAN], + ["null", IS_NULL], + ["array", IS_ARRAY], + ["object", IS_OBJECT], + ["bytes", IS_BYTES], +]); + +/** Leaf checks that object and array loops run inline by code instead of calling them. */ +export const LEAF_CODES: ReadonlyMap = new Map([ + [IS_STRING, 1], + [IS_NUMBER, 2], + [IS_INTEGER, 3], + [IS_BOOLEAN, 4], + [IS_NULL, 5], + [ALWAYS, 6], +]); + +function leafOk(code: number, x: unknown): boolean { + switch (code) { + case 1: + return typeof x === "string"; + case 2: + return typeof x === "number"; + case 3: + return Number.isInteger(x); + case 4: + return typeof x === "boolean"; + case 5: + return x === null; + default: + return true; + } +} + +/** Whether a combined object or array check also checks the type: not at all, strictly, or passing other types. */ +export const GUARD_NONE = 0; +export const GUARD_STRICT = 1; +export const GUARD_LOOSE = 2; +export type Guard = typeof GUARD_NONE | typeof GUARD_STRICT | typeof GUARD_LOOSE; + +/** Flags of a declared property in an object check's entries: its leaf code and two bits. */ +export const CODE = 7; +export const REQUIRED = 8; +export const INHERITED = 16; + +/** All of `parts`; two and three parts get closures without a loop. */ +export function and(parts: readonly Check[]): Check { + switch (parts.length) { + case 0: + return ALWAYS; + case 1: + return parts[0]; + case 2: { + const [a, b] = parts; + return (v) => a(v) && b(v); + } + case 3: { + const [a, b, c] = parts; + return (v) => a(v) && b(v) && c(v); + } + default: + return (v) => { + for (let i = 0; i < parts.length; i++) if (!parts[i](v)) return false; + return true; + }; + } +} + +/** A check whose implementation is set later: the indirection a recursive reference goes through. */ +export function indirect(slot: { check: Check | undefined }): Check { + return (v) => (slot.check as Check)(v); +} + +/** `check` with `uri` pushed onto the dynamic scope while it runs (only for schemas with `$dynamicRef`). */ +export function scopedCheck(scope: string[], uri: string, check: Check): Check { + return (v) => { + scope.push(uri); + const ok = check(v); + scope.pop(); + return ok; + }; +} + +/** Any of the named types. */ +export function typeCheck(types: readonly unknown[]): Check { + const checks = types.map((t) => (typeof t === "string" ? TYPE_CHECKS.get(t) : undefined) ?? NEVER); + if (checks.length === 1) return checks[0]; + return (v) => { + for (let i = 0; i < checks.length; i++) if (checks[i](v)) return true; + return false; + }; +} + +/** A Uint8Array within the length bounds. */ +export function bytesCheck(min: number, max: number): Check { + return (v) => v instanceof Uint8Array && v.length >= min && v.length <= max; +} + +/** The number of Unicode code points of `s`: JSON Schema string lengths count code points, not UTF-16 units. */ +export function codePoints(s: string): number { + let n = 0; + for (let i = 0; i < s.length; i++, n++) { + const c = s.charCodeAt(i); + // A high surrogate starts a pair only when a low surrogate follows; an unpaired one is a code point of its own. + if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) { + const next = s.charCodeAt(i + 1); + if (next >= 0xdc00 && next <= 0xdfff) i++; + } + } + return n; +} + +// UTF-16 units bound code points from both sides (units / 2 <= points <= units), so most strings need no count. +function lengthOk(s: string, min: number, max: number): boolean { + const units = s.length; + if (units < min) return false; + if (units < 2 * min && codePoints(s) < min) return false; + return units <= max || codePoints(s) <= max; +} + +/** The string keywords: length bounds, pattern and format. */ +export function stringCheck( + strict: boolean, + min: number, + max: number, + re: RegExp | undefined, + format: ((value: string) => boolean) | undefined, +): Check { + if (re !== undefined && min === 0 && max === Number.POSITIVE_INFINITY && format === undefined) { + return strict ? (v) => typeof v === "string" && re.test(v) : (v) => typeof v !== "string" || re.test(v); + } + if (re === undefined && format === undefined) { + return strict + ? (v) => typeof v === "string" && lengthOk(v, min, max) + : (v) => typeof v !== "string" || lengthOk(v, min, max); + } + return (v) => { + if (typeof v !== "string") return !strict; + return lengthOk(v, min, max) && (re === undefined || re.test(v)) && (format === undefined || format(v)); + }; +} + +/** The number keywords. `mode`: 0 passes other types, 1 requires a number, 2 an integer. */ +export function numberCheck( + mode: 0 | 1 | 2, + min: number, + max: number, + xmin: number, + xmax: number, + multipleOf: number | undefined, +): Check { + return (v) => { + if (typeof v !== "number") return mode === 0; + if (mode === 2 && !Number.isInteger(v)) return false; + return v >= min && v <= max && v > xmin && v < xmax && (multipleOf === undefined || isMultiple(v, multipleOf)); + }; +} + +/** Whether `v` is a multiple of `d`, exactly for decimals: 0.0075 / 0.0001 is 74.99999999999999 in floating point. */ +export function isMultiple(v: number, d: number): boolean { + const q = v / d; + if (Number.isInteger(q)) return true; + if (!Number.isFinite(q)) return v % d === 0; + const [vc, ve] = decimal(v); + const [dc, de] = decimal(d); + return ve >= de ? (vc * 10n ** BigInt(ve - de)) % dc === 0n : vc % (dc * 10n ** BigInt(de - ve)) === 0n; +} + +function decimal(x: number): [bigint, number] { + const [mantissa, exponent = "0"] = Math.abs(x).toString().split("e"); + const point = mantissa.indexOf("."); + const fraction = point < 0 ? 0 : mantissa.length - point - 1; + return [BigInt(mantissa.replace(".", "")), Number(exponent) - fraction]; +} + +/** `inner` for objects; other values pass unless `strict`. */ +export function guardObject(inner: Check, strict: boolean): Check { + return strict ? (v) => isObject(v) && inner(v) : (v) => !isObject(v) || inner(v); +} + +/** `inner` for arrays; other values pass unless `strict`. */ +export function guardArray(inner: Check, strict: boolean): Check { + return strict ? (v) => Array.isArray(v) && inner(v) : (v) => !Array.isArray(v) || inner(v); +} + +/** + * An object with declared properties: `entries` is `[key, check, flags]` per property. The value's own keys are + * walked with for...in (whose `o[k]` V8 reads from the enum cache, unlike a load by a name from a list), each found + * among the declared names; `closed` rejects any other key. A property whose value is `undefined` counts as absent. + */ +export function declaredObject( + entries: readonly unknown[], + requiredCount: number, + closed: boolean, + guard: Guard, +): Check { + const n = entries.length; + return (v) => { + if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; + // Guarded: by the check above, or by the caller for GUARD_NONE. + const o = v as Record; + let required = 0; + for (const k in o) { + let i = 0; + while (i < n && entries[i] !== k) i += 3; + if (i === n) { + if (closed) return false; + continue; + } + const x = o[k]; + if (x === undefined) continue; + const f = entries[i + 2] as number; + const c = f & CODE; + if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + if (f & REQUIRED) required++; + } + return required === requiredCount; + }; +} + +/** An object with declared properties, required keys not declared, pattern properties and additional properties. */ +export function openObject( + entries: readonly unknown[], + keys: readonly string[], + extraRequired: readonly string[], + patterns: readonly (readonly [RegExp, Check])[], + additional: Check | undefined, + guard: Guard, +): Check { + const known = patterns.length > 0 || additional !== undefined ? new Set(keys) : undefined; + return (v) => { + if (guard !== GUARD_NONE && !isObject(v)) return guard === GUARD_LOOSE; + const o = v as Record; + for (let i = 0; i < entries.length; i += 3) { + const k = entries[i] as string; + const x = o[k]; + const f = entries[i + 2] as number; + if (x === undefined || (f & INHERITED && !Object.hasOwn(o, k))) { + if (f & REQUIRED) return false; + continue; + } + const c = f & CODE; + if (c !== 0 ? !leafOk(c, x) : !(entries[i + 1] as Check)(x)) return false; + } + for (let i = 0; i < extraRequired.length; i++) if (!Object.hasOwn(o, extraRequired[i])) return false; + if (known === undefined) return true; + for (const k of Object.keys(o)) { + let matched = known.has(k); + for (let i = 0; i < patterns.length; i++) { + if (patterns[i][0].test(k)) { + matched = true; + if (!patterns[i][1](o[k])) return false; + } + } + if (!matched && additional !== undefined && !additional(o[k])) return false; + } + return true; + }; +} + +/** Every property name passes `check` (`v` is an object). */ +export function propertyNamesCheck(check: Check): Check { + return (v) => { + for (const k in v as object) if (!check(k)) return false; + return true; + }; +} + +/** `minProperties` and `maxProperties` (`v` is an object). */ +export function propertyCountCheck(min: number, max: number): Check { + return (v) => { + let n = 0; + for (const _ in v as object) n++; + return n >= min && n <= max; + }; +} + +/** `dependentRequired` (`v` is an object). */ +export function dependentRequiredCheck(entries: readonly (readonly [string, readonly string[]])[]): Check { + return (v) => { + const o = v as object; + for (const [k, list] of entries) + if (Object.hasOwn(o, k)) for (const r of list) if (!Object.hasOwn(o, r)) return false; + return true; + }; +} + +/** `dependentSchemas` (`v` is an object). */ +export function dependentSchemasCheck(entries: readonly (readonly [string, Check])[]): Check { + return (v) => { + for (const [k, check] of entries) if (Object.hasOwn(v as object, k) && !check(v)) return false; + return true; + }; +} + +/** The common array: guard, length bounds, prefix items and the items after them, in one closure. */ +export function listCheck( + guard: Guard, + min: number, + max: number, + prefix: readonly Check[], + rest: Check | undefined, +): Check { + const n = prefix.length; + const code = rest === undefined ? 0 : (LEAF_CODES.get(rest) ?? 0); + return (v) => { + if (!Array.isArray(v)) return guard === GUARD_LOOSE; + const length = v.length; + if (length < min || length > max) return false; + const m = length < n ? length : n; + for (let i = 0; i < m; i++) if (!prefix[i](v[i])) return false; + if (rest === undefined) return true; + if (code !== 0) { + for (let i = n; i < length; i++) if (!leafOk(code, v[i])) return false; + return true; + } + for (let i = n; i < length; i++) if (!rest(v[i])) return false; + return true; + }; +} + +/** Prefix items and the items after them (`v` is an array). */ +export function itemsCheck(prefix: readonly Check[], rest: Check | undefined): Check { + return listCheck(GUARD_NONE, 0, Number.POSITIVE_INFINITY, prefix, rest); +} + +/** `minItems` and `maxItems` (`v` is an array). */ +export function lengthCheck(min: number, max: number): Check { + return (v) => (v as unknown[]).length >= min && (v as unknown[]).length <= max; +} + +/** `contains` with its bounds (`v` is an array). */ +export function containsCheck(check: Check, min: number, max: number): Check { + return (v) => { + const list = v as unknown[]; + let n = 0; + for (let i = 0; i < list.length; i++) if (check(list[i]) && ++n > max) return false; + return n >= min; + }; +} + +/** `uniqueItems` (`v` is an array). */ +export function uniqueCheck(v: unknown): boolean { + const list = v as unknown[]; + for (let i = 1; i < list.length; i++) for (let j = 0; j < i; j++) if (deepEqual(list[i], list[j])) return false; + return true; +} + +/** `enum`: up to 8 primitives by a scan, more by a Set, objects and arrays by deep equality. */ +export function enumCheck(values: readonly unknown[]): Check { + const primitives = values.filter((x) => typeof x !== "object" || x === null); + if (primitives.length === values.length && primitives.length <= 8) { + return (v) => { + for (let i = 0; i < primitives.length; i++) if (primitives[i] === v) return true; + return false; + }; + } + const set = new Set(primitives); + const objects = values.filter((x) => typeof x === "object" && x !== null); + return (v) => set.has(v) || (typeof v === "object" && objects.some((o) => deepEqual(v, o))); +} + +/** `const`. */ +export function constCheck(value: unknown): Check { + if (typeof value !== "object" || value === null) return (v) => v === value; + return (v) => deepEqual(v, value); +} + +/** `anyOf`. */ +export function anyCheck(checks: readonly Check[]): Check { + return (v) => { + for (let i = 0; i < checks.length; i++) if (checks[i](v)) return true; + return false; + }; +} + +/** `oneOf`. */ +export function oneCheck(checks: readonly Check[]): Check { + return (v) => { + let n = 0; + for (let i = 0; i < checks.length; i++) if (checks[i](v) && ++n > 1) return false; + return n === 1; + }; +} + +/** `not`. */ +export function notCheck(check: Check): Check { + return (v) => !check(v); +} + +/** `if`, `then` and `else`. */ +export function ifCheck(cond: Check, then: Check, otherwise: Check): Check { + return (v) => (cond(v) ? then(v) : otherwise(v)); +} + +/** A union of objects tagged by a const on `key`: the tag selects the one branch that can match. */ +export function taggedCheck(key: string, tags: readonly unknown[], checks: readonly Check[]): Check { + return (v) => { + if (!isObject(v)) return false; + const tag = v[key]; + for (let i = 0; i < tags.length; i++) if (tags[i] === tag) return checks[i](v); + return false; + }; +} + +/** JSON equality: same primitives, arrays with equal items in order, objects with the same keys and equal values. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false; + const array = Array.isArray(a); + if (array !== Array.isArray(b)) return false; + if (array) { + const x = a as unknown[]; + const y = b as unknown[]; + if (x.length !== y.length) return false; + for (let i = 0; i < x.length; i++) if (!deepEqual(x[i], y[i])) return false; + return true; + } + const x = a as Record; + const y = b as Record; + const keys = Object.keys(x); + if (keys.length !== Object.keys(y).length) return false; + for (const k of keys) if (!Object.hasOwn(y, k) || !deepEqual(x[k], y[k])) return false; + return true; +} diff --git a/packages/schema/src/coerce.ts b/packages/schema/src/coerce.ts new file mode 100644 index 000000000..5dee2487f --- /dev/null +++ b/packages/schema/src/coerce.ts @@ -0,0 +1,556 @@ +// Coercion: turns a loosely typed value (a model's tool call arguments, form input, query strings) into the value a +// schema most likely meant, before it is checked. Only validators compiled with `{ coerce: true }` have it, and they +// build it on their first coerce() or parse(); the check path never runs it. +// +// Three passes, in this order: +// 1. Optional nulls: a `null` in an optional property whose schema rejects null is dropped, as if omitted. +// 2. Conversion rules ported from TypeBox 1.3.27's Value.Convert (MIT), keyed on JSON Schema structure (`$ref`, `const`, +// `enum`, `anyOf`/`oneOf`, `allOf`, `type`, `items`/`prefixItems`, `properties`/`patternProperties`) instead of +// TypeBox's kind marks: `null` becomes "null" for a string, `"3.5"` becomes 3 for an integer, a single value becomes +// a one-item array, and a union takes the first branch whose conversion passes. +// 3. Primitive coercion by `type` (also type arrays), as in Ajv's `coerceTypes`: what pass 2 left, for example a type +// array `["integer", "null"]`, or `""` for `null`. +// A value that cannot be coerced is left as it is; the check then reports it. +// +// Every pass is copy-on-write: an object or array is copied only when one of its members changes, so an unchanged +// value comes back as the same reference and the input is never modified. All passes walk the same keywords for +// members: `properties`, then `patternProperties`, then `additionalProperties` for keys neither names; tuple items by +// `prefixItems` (or a draft-7 `items` array), the rest by `items`. Draft 7's `additionalItems` is not coerced. + +import type { Compiler } from "./compiler.ts"; +import { isSchemaObject } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { SchemaObject } from "./types.ts"; + +/** Coerces values for one compiled schema. */ +export class Coercer { + private readonly c: Compiler; + private readonly schema: unknown; + /** + * The (schema node, value) pairs pass 2 is converting through a re-entry point, as a stack. Conversion can enter + * a node again with the same value only through a `$ref` or by wrapping a single value into an array (also for + * schemas that cycle through shared JS objects): a recursive schema that meets a value no branch accepts would + * convert it without end. A pair already on the stack (primitives by value, objects by reference) leaves the value + * unchanged. Only those two points push, so schemas without references or wraps pay nothing. + */ + private readonly entered: unknown[] = []; + /** How often a re-entry was cut short in this coerce() call; results computed across one are not memoized. */ + private cuts = 0; + /** + * Results of `$ref` targets by (node, value) within one coerce() call: a recursive union whose branches all + * convert the same children (and all fail) would convert them once per branch, exponentially in the depth. + */ + private memo: Map> | undefined = undefined; + + constructor(compiler: Compiler, schema: unknown) { + this.c = compiler; + this.schema = schema; + } + + /** `value` coerced: a copy along the changed paths only, or `value` itself when nothing changes. */ + coerce(value: unknown): unknown { + const base = this.c.r.rootBase; + const withoutNulls = this.dropOptionalNulls(value, this.schema, base); + return this.coerceTypes(this.convertOnly(withoutNulls), this.schema, base); + } + + /** Pass 2 alone, as TypeBox's `Value.Convert` (the TypeBox conformance tests run it). */ + convertOnly(value: unknown): unknown { + try { + return this.convert(this.schema, this.c.r.rootBase, value); + } finally { + this.memo = undefined; + this.cuts = 0; + } + } + + /** `node` converted for `value` at a re-entry point: unchanged when that pair is already being converted. */ + private reenter(node: object, base: string, value: unknown, memoize: boolean): unknown { + const entered = this.entered; + for (let i = 0; i < entered.length; i += 2) { + if (entered[i] === node && Object.is(entered[i + 1], value)) { + this.cuts++; + return value; + } + } + const memo = memoize ? this.memo?.get(node) : undefined; + if (memo?.has(value)) return memo.get(value); + const cuts = this.cuts; + entered.push(node, value); + let result: unknown; + try { + result = this.convert(node, base, value); + } finally { + entered.length -= 2; + } + if (memoize && this.cuts === cuts) { + this.memo ??= new Map(); + let values = this.memo.get(node); + if (values === undefined) { + values = new Map(); + this.memo.set(node, values); + } + values.set(value, result); + } + return result; + } + + /** Whether `value` passes the schema `node`; undefined when `node` cannot be compiled. */ + private passes(node: unknown, base: string, value: unknown): boolean | undefined { + try { + return this.c.check(node, this.baseOf(node, base))(value); + } catch { + return undefined; + } + } + + private baseOf(node: unknown, base: string): string { + return isSchemaObject(node) ? (this.c.r.baseOf.get(node) ?? base) : base; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 1: optional nulls + // --------------------------------------------------------------------------------------------------------------- + + private dropOptionalNulls(value: unknown, schema: unknown, base: string): unknown { + if (!isSchemaObject(schema)) return value; + const b = this.baseOf(schema, base); + if (Array.isArray(value)) { + return mapItems(value, (item, i) => { + const itemSchema = itemSchemaAt(schema, i); + return itemSchema === undefined ? item : this.dropOptionalNulls(item, itemSchema, b); + }); + } + if (!isPlainObject(value)) return value; + const properties = isSchemaObject(schema.properties) ? schema.properties : {}; + const patterns = isSchemaObject(schema.patternProperties) + ? Object.entries(schema.patternProperties).map(([pattern, item]) => [sharedRegex(pattern), item] as const) + : []; + const additional = isSchemaObject(schema.additionalProperties) ? schema.additionalProperties : undefined; + if (patterns.length === 0 && additional === undefined && !isSchemaObject(schema.properties)) return value; + const required = new Set(Array.isArray(schema.required) ? schema.required : []); + let out: Record | undefined; + const set = (key: string, next: unknown) => { + if (next === (out ?? value)[key]) return; + out ??= { ...value }; + out[key] = next; + }; + for (const [key, property] of Object.entries(properties)) { + if (!Object.hasOwn(value, key)) continue; + const isRef = isSchemaObject(property) && typeof property.$ref === "string"; + if (value[key] === null && !required.has(key) && !isRef && this.passes(property, b, null) === false) { + out ??= { ...value }; + delete out[key]; + continue; + } + set(key, this.dropOptionalNulls(value[key], property, b)); + } + // Values of pattern and additional properties are walked for their own optional nulls; the keys themselves + // are not optional properties, so they stay. + if (patterns.length > 0 || additional !== undefined) { + for (const key of Object.keys(value)) { + let matched = Object.hasOwn(properties, key); + for (const [pattern, item] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + set(key, this.dropOptionalNulls((out ?? value)[key], item, b)); + } + if (!matched && additional !== undefined) set(key, this.dropOptionalNulls(value[key], additional, b)); + } + } + return out ?? value; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 2: TypeBox's conversion rules + // --------------------------------------------------------------------------------------------------------------- + + private convert(node: unknown, base: string, value: unknown): unknown { + if (!isSchemaObject(node)) return value; + const b = this.baseOf(node, base); + if (typeof node.$ref === "string") { + let target: { node: unknown; base: string }; + try { + target = this.c.r.resolve(node.$ref, b); + } catch { + return value; + } + return isSchemaObject(target.node) ? this.reenter(target.node, target.base, value, true) : value; + } + return this.convertNode(node, b, value); + } + + /** One schema node's conversion: its first keyword among `const`, `enum`, the unions, `allOf` and `type`. */ + private convertNode(node: SchemaObject, b: string, value: unknown): unknown { + if (Object.hasOwn(node, "const")) return convertLiteral(node.const, value); + if (Array.isArray(node.enum)) return this.convertEnum(node, node.enum, b, value); + // A union or intersection node may also declare members (plain JSON Schema; TypeBox's never do): they convert + // after the branches. + if (Array.isArray(node.anyOf)) return this.convertUnion(node, node.anyOf, b, value); + if (Array.isArray(node.oneOf)) return this.convertUnion(node, node.oneOf, b, value); + if (Array.isArray(node.allOf)) { + let next = value; + for (const member of node.allOf) next = this.convert(member, b, next); + return this.convertMembers(node, b, next); + } + switch (node.type) { + case "string": + return orValue(tryString(value), value); + case "number": + return orValue(tryNumber(value), value); + case "integer": { + const result = tryNumber(value); + return result === undefined ? value : Math.trunc(result.value); + } + case "boolean": + return orValue(tryBoolean(value), value); + case "null": + return orValue(tryNull(value), value); + case "array": + return this.convertArray(node, b, value); + case "object": + return this.convertObject(node, b, value); + case undefined: + if (node.properties !== undefined || node.patternProperties !== undefined) { + return this.convertObject(node, b, value); + } + if (node.items !== undefined || node.prefixItems !== undefined) return this.convertArray(node, b, value); + return value; + default: + return value; + } + } + + /** An enum is a union of its values: a member stays, else the first value the input converts to. */ + private convertEnum(node: SchemaObject, values: readonly unknown[], base: string, value: unknown): unknown { + if (values.some((member) => member === value)) return value; + for (const member of values) { + const candidate = convertLiteral(member, value); + if (candidate !== value && this.passes(node, base, candidate) === true) return candidate; + } + return value; + } + + /** + * A value one branch accepts stays; else the first branch's conversion that the whole union accepts. Branches are + * converted in order and only until one is accepted, so a later recursive branch is not entered needlessly. A node + * that also declares members (plain JSON Schema; TypeBox's never do) converts them into each candidate before it is + * checked, so a branch is not discarded for a sibling property that is converted after it. + */ + private convertUnion(node: SchemaObject, branches: readonly unknown[], base: string, value: unknown): unknown { + if (branches.some((branch) => this.passes(branch, base, value) === true)) + return this.convertMembers(node, base, value); + for (const branch of branches) { + const candidate = this.convertMembers(node, base, this.convert(branch, base, value)); + if (this.passes(node, base, candidate) === true) return candidate; + } + return this.convertMembers(node, base, value); + } + + /** The object members of a union or intersection node, when it declares any. */ + private convertMembers(node: SchemaObject, base: string, value: unknown): unknown { + const declares = + node.properties !== undefined || + node.patternProperties !== undefined || + node.additionalProperties !== undefined; + return declares ? this.convertObject(node, base, value) : value; + } + + private convertArray(node: SchemaObject, base: string, value: unknown): unknown { + const prefix = Array.isArray(node.items) + ? node.items + : Array.isArray(node.prefixItems) + ? node.prefixItems + : undefined; + if (prefix !== undefined) { + // A tuple converts its items and never wraps a single value. + if (!Array.isArray(value)) return value; + const rest = Array.isArray(node.items) ? undefined : node.items; + return mapItems(value, (item, i) => { + const schema = i < prefix.length ? prefix[i] : rest; + return schema === undefined ? item : this.convert(schema, base, item); + }); + } + if (Array.isArray(value)) { + return node.items === undefined ? value : mapItems(value, (item) => this.convert(node.items, base, item)); + } + // The wrapped single value is converted by the item schema with the same value: a re-entry point. + const item = node.items; + return [isSchemaObject(item) ? this.reenter(item, base, value, false) : value]; + } + + private convertObject(node: SchemaObject, base: string, value: unknown): unknown { + if (!isPlainObject(value)) return value; + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const patterns = isSchemaObject(node.patternProperties) + ? Object.entries(node.patternProperties).map(([pattern, schema]) => [sharedRegex(pattern), schema] as const) + : []; + const additional = isSchemaObject(node.additionalProperties) ? node.additionalProperties : undefined; + let out: Record | undefined; + for (const key of Object.keys(value)) { + // An undefined property counts as absent. + if (value[key] === undefined) continue; + let next: unknown = value[key]; + let matched = Object.hasOwn(properties, key); + if (matched) next = this.convert(properties[key], base, next); + for (const [pattern, schema] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + next = this.convert(schema, base, next); + } + if (!matched && additional !== undefined) next = this.convert(additional, base, next); + if (next !== value[key]) { + out ??= { ...value }; + out[key] = next; + } + } + return out ?? value; + } + + // --------------------------------------------------------------------------------------------------------------- + // Pass 3: primitive coercion by type + // --------------------------------------------------------------------------------------------------------------- + + private coerceTypes(value: unknown, schema: unknown, base: string): unknown { + if (!isSchemaObject(schema)) return value; + const b = this.baseOf(schema, base); + let next = value; + if (Array.isArray(schema.allOf)) for (const member of schema.allOf) next = this.coerceTypes(next, member, b); + if (Array.isArray(schema.anyOf)) next = this.coerceUnion(next, schema.anyOf, b); + if (Array.isArray(schema.oneOf)) next = this.coerceUnion(next, schema.oneOf, b); + const types = typesOf(schema); + const matchesUnionMember = types.length > 1 && types.some((type) => matchesType(next, type)); + if (types.length > 0 && !matchesUnionMember) { + for (const type of types) { + const candidate = coercePrimitive(next, type); + if (candidate !== next) { + next = candidate; + break; + } + } + } + if (types.includes("object") && isPlainObject(next)) next = this.coerceProperties(next, schema, b); + if (types.includes("array") && Array.isArray(next)) next = this.coerceItems(next, schema, b); + return next; + } + + private coerceUnion(value: unknown, branches: readonly unknown[], base: string): unknown { + for (const branch of branches) if (this.passes(branch, base, value) === true) return value; + for (const branch of branches) { + const coerced = this.coerceTypes(value, branch, base); + if (this.passes(branch, base, coerced) === true) return coerced; + } + return value; + } + + private coerceProperties( + value: Record, + schema: SchemaObject, + base: string, + ): Record { + const properties = isSchemaObject(schema.properties) ? schema.properties : {}; + const patterns = isSchemaObject(schema.patternProperties) + ? Object.entries(schema.patternProperties).map(([pattern, item]) => [sharedRegex(pattern), item] as const) + : []; + const additional = isSchemaObject(schema.additionalProperties) ? schema.additionalProperties : undefined; + let out: Record | undefined; + const set = (key: string, next: unknown) => { + if (next === (out ?? value)[key]) return; + out ??= { ...value }; + out[key] = next; + }; + for (const [key, property] of Object.entries(properties)) { + if (key in value) set(key, this.coerceTypes(value[key], property, base)); + } + for (const key of Object.keys(value)) { + let matched = Object.hasOwn(properties, key); + for (const [pattern, item] of patterns) { + if (!pattern.test(key)) continue; + matched = true; + set(key, this.coerceTypes((out ?? value)[key], item, base)); + } + if (!matched && additional !== undefined) set(key, this.coerceTypes(value[key], additional, base)); + } + return out ?? value; + } + + private coerceItems(value: unknown[], schema: SchemaObject, base: string): unknown[] { + return mapItems(value, (item, i) => { + const itemSchema = itemSchemaAt(schema, i); + return isSchemaObject(itemSchema) ? this.coerceTypes(item, itemSchema, base) : item; + }); + } +} + +/** + * The schema of an array's item at `index`: a tuple's (`prefixItems`, or a draft-7 `items` array) for its prefix, + * `items` after `prefixItems`, or `items` for every item. Undefined when nothing applies. + */ +function itemSchemaAt(schema: SchemaObject, index: number): unknown { + if (Array.isArray(schema.items)) return schema.items[index]; + if (Array.isArray(schema.prefixItems) && index < schema.prefixItems.length) return schema.prefixItems[index]; + return schema.items; +} + +/** `items` mapped by `map`; the same array when every item maps to itself. */ +function mapItems(items: unknown[], map: (item: unknown, index: number) => unknown): unknown[] { + let out: unknown[] | undefined; + for (let i = 0; i < items.length; i++) { + const next = map(items[i], i); + if (next === items[i]) continue; + out ??= items.slice(); + out[i] = next; + } + return out ?? items; +} + +// ------------------------------------------------------------------------------------------------------------------- +// TypeBox's conversions of one value (value/convert/try in TypeBox 1.3.27): a result, or undefined when there is none +// ------------------------------------------------------------------------------------------------------------------- + +type Converted = { readonly value: T } | undefined; + +function orValue(result: Converted, value: unknown): unknown { + return result === undefined ? value : result.value; +} + +/** A literal (`const`, an enum value) taken from a value of another type that converts to exactly it. */ +function convertLiteral(literal: unknown, value: unknown): unknown { + if (literal === value) return value; + const result = + typeof literal === "string" + ? tryString(value) + : typeof literal === "number" + ? tryNumber(value) + : typeof literal === "boolean" + ? tryBoolean(value) + : undefined; + return result !== undefined && result.value === literal ? result.value : value; +} + +function tryString(value: unknown): Converted { + if (typeof value === "bigint" || typeof value === "boolean" || isFiniteNumber(value)) + return { value: String(value) }; + if (value === null) return { value: "null" }; + if (typeof value === "string") return { value }; + if (value === undefined) return { value: "" }; + return undefined; +} + +const MAX_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); +const MIN_BIGINT = BigInt(Number.MIN_SAFE_INTEGER); + +function tryNumber(value: unknown): Converted { + if (typeof value === "bigint") + return value <= MAX_BIGINT && value >= MIN_BIGINT ? { value: Number(value) } : undefined; + if (typeof value === "boolean") return { value: value ? 1 : 0 }; + if (isFiniteNumber(value)) return { value }; + if (value === null || value === undefined) return { value: 0 }; + if (typeof value !== "string") return undefined; + const coerced = +value; + if (Number.isFinite(coerced)) return { value: coerced }; + const lowercase = value.toLowerCase(); + if (lowercase === "false") return { value: 0 }; + if (lowercase === "true") return { value: 1 }; + const big = bigIntOfString(value); + return big !== undefined && big <= MAX_BIGINT && big >= MIN_BIGINT ? { value: Number(big) } : undefined; +} + +/** TypeBox's TryBigInt of a string: `12n`, `12.5` (truncated) or `12`; true and false are not reached from tryNumber. */ +function bigIntOfString(value: string): bigint | undefined { + if (/^-?(0|[1-9]\d*)n$/.test(value)) return BigInt(value.slice(0, -1)); + if (/^-?(0|[1-9]\d*)\.\d+$/.test(value)) return BigInt(value.split(".")[0]); + if (/^-?(0|[1-9]\d*)$/.test(value)) return BigInt(value); + return undefined; +} + +function tryBoolean(value: unknown): Converted { + if (typeof value === "bigint") return value === 0n ? { value: false } : value === 1n ? { value: true } : undefined; + if (typeof value === "boolean") return { value }; + if (isFiniteNumber(value)) return value === 0 ? { value: false } : value === 1 ? { value: true } : undefined; + if (value === null || value === undefined) return { value: false }; + if (typeof value !== "string") return undefined; + const lowercase = value.toLowerCase(); + if (lowercase === "false" || value === "0") return { value: false }; + if (lowercase === "true" || value === "1") return { value: true }; + return undefined; +} + +function tryNull(value: unknown): Converted { + if (typeof value === "bigint") return value === 0n ? { value: null } : undefined; + if (typeof value === "boolean") return value === false ? { value: null } : undefined; + if (isFiniteNumber(value)) return value === 0 ? { value: null } : undefined; + if (value === null || value === undefined) return { value: null }; + if (typeof value !== "string") return undefined; + const lowercase = value.toLowerCase(); + return lowercase === "undefined" || lowercase === "null" || value === "" || value === "0" + ? { value: null } + : undefined; +} + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +// ------------------------------------------------------------------------------------------------------------------- +// Primitive coercion by type (pass 3) +// ------------------------------------------------------------------------------------------------------------------- + +function typesOf(schema: SchemaObject): string[] { + if (typeof schema.type === "string") return [schema.type]; + if (Array.isArray(schema.type)) return schema.type.filter((type): type is string => typeof type === "string"); + return []; +} + +function matchesType(value: unknown, type: string): boolean { + switch (type) { + case "number": + return typeof value === "number"; + case "integer": + return typeof value === "number" && Number.isInteger(value); + case "boolean": + return typeof value === "boolean"; + case "string": + return typeof value === "string"; + case "null": + return value === null; + case "array": + return Array.isArray(value); + case "object": + return isPlainObject(value); + default: + return false; + } +} + +function coercePrimitive(value: unknown, type: string): unknown { + switch (type) { + case "number": + case "integer": { + if (value === null) return 0; + if (typeof value === "string" && value.trim() !== "") { + const parsed = Number(value); + if (type === "number" ? Number.isFinite(parsed) : Number.isInteger(parsed)) return parsed; + } + if (typeof value === "boolean") return value ? 1 : 0; + return value; + } + case "boolean": + if (value === null) return false; + if (value === "true" || value === 1) return true; + if (value === "false" || value === 0) return false; + return value; + case "string": + if (value === null) return ""; + if (typeof value === "number" || typeof value === "boolean") return String(value); + return value; + case "null": + return value === "" || value === 0 || value === false ? null : value; + default: + return value; + } +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/schema/src/compiler.ts b/packages/schema/src/compiler.ts new file mode 100644 index 000000000..8140e38d5 --- /dev/null +++ b/packages/schema/src/compiler.ts @@ -0,0 +1,593 @@ +// The compiler: a schema to one check closure, built once. Keywords group by the type they constrain, so a value +// is classified once per schema, and the common shapes (closed objects, arrays of one item schema, tagged unions, +// leaf schemas) get one specialized closure each. + +import { + ANNOTATING_FALSE, + ANNOTATING_TRUE, + type AnnotatingCheck, + annotateAll, + annotateAny, + annotateArray, + annotateDependent, + annotateIf, + annotateObject, + ignoreAnnotations, + indirectAnnotating, + isolateAnnotations, + scopedAnnotating, + unevaluatedItems, + unevaluatedProperties, + withFreshAnnotations, +} from "./annotations.ts"; +import { + ALWAYS, + and, + anyCheck, + bytesCheck, + constCheck, + containsCheck, + declaredObject, + dependentRequiredCheck, + dependentSchemasCheck, + enumCheck, + GUARD_LOOSE, + GUARD_NONE, + GUARD_STRICT, + type Guard, + guardArray, + guardObject, + INHERITED, + IS_ARRAY, + IS_OBJECT, + ifCheck, + indirect, + itemsCheck, + LEAF_CODES, + lengthCheck, + listCheck, + NEVER, + notCheck, + numberCheck, + oneCheck, + openObject, + propertyCountCheck, + propertyNamesCheck, + REQUIRED, + scopedCheck, + stringCheck, + TYPE_CHECKS, + taggedCheck, + typeCheck, + uniqueCheck, +} from "./checks.ts"; +import { documentOf, isSchemaObject, Resolver, resolveUri } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { Check, CompileOptions, FormatCheck, JsonSchema, SchemaObject } from "./types.ts"; + +/** Thrown when a compile without dynamic scope tracking meets a bookended `$dynamicRef`: compile again with it. */ +export const RESTART = Symbol("restart"); + +/** One compile of a schema. Its caches and indexes are garbage once the root check is built. */ +export class Compiler { + readonly r: Resolver; + readonly draft: "2020-12" | "7"; + readonly formats: Readonly> | undefined; + /** Whether checks track the dynamic scope (the schema has a bookended `$dynamicRef`). */ + readonly dynamic: boolean; + private readonly schema: JsonSchema; + private readonly scope: string[]; + private readonly cache = new Map(); + private readonly annotatingCache = new Map(); + + constructor(schema: JsonSchema, options: CompileOptions, dynamic: boolean) { + this.schema = schema; + this.r = new Resolver(schema, options); + this.draft = this.r.draft; + this.formats = options.formats; + this.dynamic = dynamic; + this.scope = []; + } + + /** The check of the root schema. */ + root(): Check { + return this.check(this.schema, this.r.rootBase); + } + + /** The check of a schema (cached by schema object; a recursive reference goes through an indirection). */ + check(node: unknown, base: string): Check { + if (node === true) return ALWAYS; + if (node === false) return NEVER; + if (!isSchemaObject(node)) + throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); + const cached = this.cache.get(node); + if (cached !== undefined) return cached; + const bare = this.bareType(node); + if (bare !== undefined) return bare; + const slot: { check: Check | undefined } = { check: undefined }; + this.cache.set(node, indirect(slot)); + const check = this.build(node, this.r.baseFor(node, base)); + slot.check = check; + this.cache.set(node, check); + return check; + } + + private build(node: SchemaObject, base: string): Check { + if (this.draft === "7" && typeof node.$ref === "string") return this.ref(node.$ref, base); + if (this.draft !== "7" && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { + return withFreshAnnotations(this.annotating(node, base)); + } + const check = this.plain(node, base); + return this.dynamic && this.r.isResource(node, base) ? scopedCheck(this.scope, base, check) : check; + } + + /** + * The module-level check of a schema that is only a `type` (with annotations), `ALWAYS` for one with no assertion + * at all; undefined for every other schema. These are the most common subschemas and cost nothing to share. + */ + private bareType(node: SchemaObject): Check | undefined { + let type: unknown; + for (const k in node) { + if (k === "type") type = node.type; + else if (!NEUTRAL_KEYWORDS.has(k)) return undefined; + } + if (type === undefined) return ALWAYS; + return typeof type === "string" ? TYPE_CHECKS.get(type) : undefined; + } + + /** Every keyword of `node`, not tracking annotations. */ + private plain(node: SchemaObject, base: string): Check { + const parts: Check[] = []; + const types = node.type === undefined ? undefined : Array.isArray(node.type) ? node.type : [node.type]; + const single = types?.length === 1 ? types[0] : undefined; + let typeInGroup = false; + if (single === "bytes") { + parts.push(bytesCheck(number(node.minLength) ?? 0, number(node.maxLength) ?? Number.POSITIVE_INFINITY)); + typeInGroup = true; + } else { + const string = this.stringGroup(node, single === "string"); + if (string !== undefined) parts.push(string); + const numeric = single === "number" || single === "integer"; + const number = this.numberGroup(node, single === "number" ? 1 : single === "integer" ? 2 : 0); + if (number !== undefined) parts.push(number); + const object = this.objectGroup(node, base, single === "object"); + if (object !== undefined) parts.push(object); + const array = this.arrayGroup(node, base, single === "array"); + if (array !== undefined) parts.push(array); + typeInGroup = + (single === "string" && string !== undefined) || + (numeric && number !== undefined) || + (single === "object" && object !== undefined) || + (single === "array" && array !== undefined); + } + if (types !== undefined && !typeInGroup) parts.unshift(typeCheck(types)); + this.inPlace(node, base, parts); + return and(parts); + } + + // The keywords that apply to the value itself, whatever its type. + private inPlace(node: SchemaObject, base: string, parts: Check[]): void { + if (Array.isArray(node.enum)) parts.push(enumCheck(node.enum)); + if (Object.hasOwn(node, "const")) parts.push(constCheck(node.const)); + if (Array.isArray(node.allOf)) parts.push(and(node.allOf.map((s) => this.check(s, base)))); + if (Array.isArray(node.anyOf)) parts.push(this.union(node.anyOf, base, false)); + if (Array.isArray(node.oneOf)) parts.push(this.union(node.oneOf, base, true)); + if (node.not !== undefined) parts.push(notCheck(this.check(node.not, base))); + if (node.if !== undefined && (node.then !== undefined || node.else !== undefined)) { + const then = node.then === undefined ? ALWAYS : this.check(node.then, base); + const otherwise = node.else === undefined ? ALWAYS : this.check(node.else, base); + parts.push(ifCheck(this.check(node.if, base), then, otherwise)); + } + if (typeof node.$ref === "string") parts.push(this.ref(node.$ref, base)); + if (this.draft !== "7" && typeof node.$dynamicRef === "string") { + parts.push(this.dynamicRef(node.$dynamicRef, base)); + } + } + + private union(branches: readonly unknown[], base: string, exactlyOne: boolean): Check { + const checks = () => branches.map((s) => this.check(s, base)); + return ( + constUnion(branches, exactlyOne) ?? + this.tagged(branches, base) ?? + (exactlyOne ? oneCheck(checks()) : anyCheck(checks())) + ); + } + + private ref(ref: string, base: string): Check { + const target = this.r.resolve(ref, base); + const check = this.check(target.node, target.base); + return this.dynamic && target.base !== base ? scopedCheck(this.scope, target.base, check) : check; + } + + private dynamicRef(ref: string, base: string): Check { + const resolved = this.bookended(ref, base); + if (resolved === undefined) return this.ref(ref, base); + const { initial, name } = resolved; + const fallback = this.check(initial.node, initial.base); + const lookup = this.dynamicLookup(name, (node, b) => this.check(node, b)); + const scope = this.scope; + return (v) => { + for (let i = 0; i < scope.length; i++) { + const check = lookup(scope[i]); + if (check !== null) return check(v); + } + return fallback(v); + }; + } + + /** The initial target and anchor name of a `$dynamicRef` that resolves dynamically, or undefined for a plain ref. */ + private bookended( + ref: string, + base: string, + ): { initial: { node: unknown; base: string }; name: string } | undefined { + const initial = this.r.resolve(ref, base); + const uri = resolveUri(ref, base); + const name = uri.slice(documentOf(uri).length + 1); + const anchored = isSchemaObject(initial.node) && initial.node.$dynamicAnchor === name; + if (name === "" || name.startsWith("/") || !anchored) return undefined; + if (!this.dynamic) throw RESTART; + return { initial, name }; + } + + // Resolved at run time against the dynamic scope, so a schema with a bookended $dynamicRef keeps its compiler. + private dynamicLookup(name: string, compile: (node: SchemaObject, base: string) => T): (uri: string) => T | null { + const found = new Map(); + return (uri) => { + const key = `${uri}#${name}`; + let check = found.get(key); + if (check === undefined) { + const target = this.r.dynamicAnchors.get(key); + check = target === undefined ? null : compile(target, this.r.baseOf.get(target) ?? uri); + found.set(key, check); + } + return check; + }; + } + + /** anyOf/oneOf whose branches are all objects with a distinct required const on one key: the tag picks the branch. */ + private tagged(branches: readonly unknown[], base: string): Check | undefined { + if (branches.length < 2) return undefined; + const resolved: SchemaObject[] = []; + for (const branch of branches) { + if (!isSchemaObject(branch)) return undefined; + const ref = typeof branch.$ref === "string" && Object.keys(branch).length === 1 ? branch.$ref : undefined; + const node = ref === undefined ? branch : this.r.resolve(ref, this.r.baseFor(branch, base)).node; + if (!isSchemaObject(node)) return undefined; + resolved.push(node); + } + const first = resolved[0].properties; + if (!isSchemaObject(first)) return undefined; + for (const key of Object.keys(first)) { + const tags = resolved.map((s) => tagOf(s, key)); + if (tags.some((t) => t === undefined) || new Set(tags).size !== tags.length) continue; + return taggedCheck( + key, + tags, + branches.map((s) => this.check(s, base)), + ); + } + return undefined; + } + + private stringGroup(node: SchemaObject, strict: boolean): Check | undefined { + const min = number(node.minLength) ?? 0; + const max = number(node.maxLength) ?? Number.POSITIVE_INFINITY; + const re = typeof node.pattern === "string" ? sharedRegex(node.pattern) : undefined; + const format = + this.formats !== undefined && typeof node.format === "string" && Object.hasOwn(this.formats, node.format) + ? this.formats[node.format] + : undefined; + if (min === 0 && max === Number.POSITIVE_INFINITY && re === undefined && format === undefined) return undefined; + return stringCheck(strict, min, max, re, format); + } + + private numberGroup(node: SchemaObject, mode: 0 | 1 | 2): Check | undefined { + const min = number(node.minimum); + const max = number(node.maximum); + const xmin = number(node.exclusiveMinimum); + const xmax = number(node.exclusiveMaximum); + const multipleOf = number(node.multipleOf); + if ([min, max, xmin, xmax, multipleOf].every((x) => x === undefined)) return undefined; + const inf = Number.POSITIVE_INFINITY; + return numberCheck(mode, min ?? -inf, max ?? inf, xmin ?? -inf, xmax ?? inf, multipleOf); + } + + private objectGroup(node: SchemaObject, base: string, strict: boolean): Check | undefined { + const onlyCore = OBJECT_EXTRAS.every((k) => node[k] === undefined); + // A schema with only the core object keywords gets one closure that also does the type guard. + const core = this.objectCore(node, base, onlyCore ? (strict ? GUARD_STRICT : GUARD_LOOSE) : GUARD_NONE); + if (core !== undefined && onlyCore) return core; + const parts: Check[] = core === undefined ? [] : [core]; + if (node.propertyNames !== undefined) parts.push(propertyNamesCheck(this.check(node.propertyNames, base))); + const minProperties = number(node.minProperties); + const maxProperties = number(node.maxProperties); + if (minProperties !== undefined || maxProperties !== undefined) { + parts.push(propertyCountCheck(minProperties ?? 0, maxProperties ?? Number.POSITIVE_INFINITY)); + } + const dependentRequired: [string, string[]][] = []; + const dependentSchemas: [string, Check][] = []; + if (this.draft !== "7") { + for (const [k, list] of entries(node.dependentRequired)) + if (Array.isArray(list)) dependentRequired.push([k, list]); + for (const [k, s] of entries(node.dependentSchemas)) dependentSchemas.push([k, this.check(s, base)]); + } + for (const [k, d] of entries(node.dependencies)) { + if (Array.isArray(d)) dependentRequired.push([k, d]); + else dependentSchemas.push([k, this.check(d, base)]); + } + if (dependentRequired.length > 0) parts.push(dependentRequiredCheck(dependentRequired)); + if (dependentSchemas.length > 0) parts.push(dependentSchemasCheck(dependentSchemas)); + if (parts.length === 0) return strict ? IS_OBJECT : undefined; + return guardObject(and(parts), strict); + } + + private objectCore(node: SchemaObject, base: string, guard: Guard): Check | undefined { + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const required = Array.isArray(node.required) ? node.required.filter((k) => typeof k === "string") : []; + const patterns = isSchemaObject(node.patternProperties) ? node.patternProperties : {}; + const additional = node.additionalProperties; + const keys = Object.keys(properties); + const patternKeys = Object.keys(patterns); + if ( + keys.length + required.length + patternKeys.length === 0 && + (additional === undefined || additional === true) + ) { + return undefined; + } + // One array per object: key, check, flags (leaf code, required, key also names an Object.prototype member). + const entries: unknown[] = []; + for (const k of keys) { + const check = this.check(properties[k], base); + const flags = + (LEAF_CODES.get(check) ?? 0) | + (required.includes(k) ? REQUIRED : 0) | + (k in Object.prototype ? INHERITED : 0); + entries.push(k, check, flags); + } + const extraRequired = required.filter((k) => !Object.hasOwn(properties, k)); + const pats = patternKeys.map((p) => [sharedRegex(p), this.check(patterns[p], base)] as const); + const extra = additional === undefined || additional === true ? undefined : this.check(additional, base); + if (pats.length === 0 && extraRequired.length === 0 && (extra === undefined || additional === false)) { + const requiredCount = keys.filter((k) => required.includes(k)).length; + return declaredObject(entries, requiredCount, additional === false, guard); + } + return openObject(entries, keys, extraRequired, pats, extra, guard); + } + + private arrayGroup(node: SchemaObject, base: string, strict: boolean): Check | undefined { + let prefix: readonly unknown[] = []; + let rest: unknown; + if (Array.isArray(node.items)) { + prefix = node.items; + rest = node.additionalItems; + } else { + if (this.draft !== "7" && Array.isArray(node.prefixItems)) prefix = node.prefixItems; + rest = node.items; + } + const prefixChecks = prefix.map((s) => this.check(s, base)); + const restCheck = rest === undefined || rest === true ? undefined : this.check(rest, base); + const minItems = number(node.minItems); + const maxItems = number(node.maxItems); + if ( + node.contains === undefined && + node.uniqueItems !== true && + (prefixChecks.length > 0 || restCheck !== undefined) + ) { + // The common array: one closure for the guard, the length bounds and the items. + const guard = strict ? GUARD_STRICT : GUARD_LOOSE; + return listCheck(guard, minItems ?? 0, maxItems ?? Number.POSITIVE_INFINITY, prefixChecks, restCheck); + } + const parts: Check[] = []; + if (prefixChecks.length > 0 || restCheck !== undefined) parts.push(itemsCheck(prefixChecks, restCheck)); + if (minItems !== undefined || maxItems !== undefined) { + parts.push(lengthCheck(minItems ?? 0, maxItems ?? Number.POSITIVE_INFINITY)); + } + if (node.contains !== undefined) { + const min = this.draft === "7" ? 1 : (number(node.minContains) ?? 1); + const max = + this.draft === "7" ? Number.POSITIVE_INFINITY : (number(node.maxContains) ?? Number.POSITIVE_INFINITY); + parts.push(containsCheck(this.check(node.contains, base), min, max)); + } + if (node.uniqueItems === true) parts.push(uniqueCheck); + if (parts.length === 0) return strict ? IS_ARRAY : undefined; + return guardArray(and(parts), strict); + } + + /** The annotating check of a schema evaluated in place (only below `unevaluatedProperties`/`unevaluatedItems`). */ + annotating(node: unknown, base: string): AnnotatingCheck { + if (node === true) return ANNOTATING_TRUE; + if (node === false) return ANNOTATING_FALSE; + if (!isSchemaObject(node)) + throw new Error(`A schema must be an object or a boolean, got ${JSON.stringify(node)}`); + const cached = this.annotatingCache.get(node); + if (cached !== undefined) return cached; + const slot: { check: AnnotatingCheck | undefined } = { check: undefined }; + this.annotatingCache.set(node, indirectAnnotating(slot)); + const check = this.buildAnnotating(node, this.r.baseFor(node, base)); + slot.check = check; + this.annotatingCache.set(node, check); + return check; + } + + private buildAnnotating(node: SchemaObject, base: string): AnnotatingCheck { + const rest: Record = {}; + let annotates = false; + for (const k of Object.keys(node)) { + if (ANNOTATING.has(k)) annotates = true; + else rest[k] = node[k]; + } + const plain = this.plain(rest, base); + const parts: AnnotatingCheck[] = []; + if (annotates) this.annotatingParts(node, base, parts); + const all = parts.length === 0 ? ignoreAnnotations(plain) : annotateAll(plain, parts); + const own = + node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined + ? isolateAnnotations(all) + : all; + return this.dynamic && this.r.isResource(node, base) ? scopedAnnotating(this.scope, base, own) : own; + } + + private annotatingParts(node: SchemaObject, base: string, parts: AnnotatingCheck[]): void { + if ( + node.properties !== undefined || + node.patternProperties !== undefined || + node.additionalProperties !== undefined + ) { + const props = entries(node.properties).map(([k, s]) => [k, this.check(s, base)] as const); + const pats = entries(node.patternProperties).map(([p, s]) => [sharedRegex(p), this.check(s, base)] as const); + const extra = + node.additionalProperties === undefined ? undefined : this.check(node.additionalProperties, base); + parts.push(annotateObject(props, pats, extra)); + } + if (node.prefixItems !== undefined || node.items !== undefined || node.contains !== undefined) { + const prefix = Array.isArray(node.prefixItems) + ? node.prefixItems + : Array.isArray(node.items) + ? node.items + : []; + const rest = Array.isArray(node.items) ? node.additionalItems : node.items; + parts.push( + annotateArray( + prefix.map((s) => this.check(s, base)), + rest === undefined ? undefined : this.check(rest, base), + node.contains === undefined ? undefined : this.check(node.contains, base), + number(node.minContains) ?? 1, + number(node.maxContains) ?? Number.POSITIVE_INFINITY, + ), + ); + } + if (Array.isArray(node.allOf)) for (const s of node.allOf) parts.push(this.annotating(s, base)); + if (Array.isArray(node.anyOf)) + parts.push( + annotateAny( + node.anyOf.map((s) => this.annotating(s, base)), + false, + ), + ); + if (Array.isArray(node.oneOf)) + parts.push( + annotateAny( + node.oneOf.map((s) => this.annotating(s, base)), + true, + ), + ); + if (node.if !== undefined) { + const then = node.then === undefined ? undefined : this.annotating(node.then, base); + const otherwise = node.else === undefined ? undefined : this.annotating(node.else, base); + parts.push(annotateIf(this.annotating(node.if, base), then, otherwise)); + } + if (typeof node.$ref === "string") parts.push(this.annotatingRef(node.$ref, base)); + if (typeof node.$dynamicRef === "string") parts.push(this.annotatingDynamicRef(node.$dynamicRef, base)); + for (const [k, s] of entries(node.dependentSchemas)) parts.push(annotateDependent(k, this.annotating(s, base))); + if (node.unevaluatedProperties !== undefined) { + parts.push(unevaluatedProperties(this.check(node.unevaluatedProperties, base))); + } + if (node.unevaluatedItems !== undefined) parts.push(unevaluatedItems(this.check(node.unevaluatedItems, base))); + } + + private annotatingRef(ref: string, base: string): AnnotatingCheck { + const target = this.r.resolve(ref, base); + const check = this.annotating(target.node, target.base); + return this.dynamic && target.base !== base ? scopedAnnotating(this.scope, target.base, check) : check; + } + + private annotatingDynamicRef(ref: string, base: string): AnnotatingCheck { + const resolved = this.bookended(ref, base); + if (resolved === undefined) return this.annotatingRef(ref, base); + const { initial, name } = resolved; + const fallback = this.annotating(initial.node, initial.base); + const lookup = this.dynamicLookup(name, (node, b) => this.annotating(node, b)); + const scope = this.scope; + return (v, a) => { + for (let i = 0; i < scope.length; i++) { + const check = lookup(scope[i]); + if (check !== null) return check(v, a); + } + return fallback(v, a); + }; + } +} + +/** Keywords that never affect a check. */ +export const NEUTRAL_KEYWORDS = new Set([ + "$schema", + "$comment", + "title", + "description", + "default", + "examples", + "deprecated", + "readOnly", + "writeOnly", + "$defs", + "definitions", + "contentEncoding", + "contentMediaType", + "minContains", + "maxContains", + "$vocabulary", +]); + +// The object keywords besides the core (properties, required, patternProperties, additionalProperties). +const OBJECT_EXTRAS = [ + "propertyNames", + "minProperties", + "maxProperties", + "dependentRequired", + "dependentSchemas", + "dependencies", +]; + +// The keywords an annotating check handles itself; every other keyword compiles into its plain part. +const ANNOTATING = new Set([ + "properties", + "patternProperties", + "additionalProperties", + "prefixItems", + "items", + "additionalItems", + "contains", + "minContains", + "maxContains", + "allOf", + "anyOf", + "oneOf", + "if", + "then", + "else", + "$ref", + "$dynamicRef", + "dependentSchemas", + "unevaluatedProperties", + "unevaluatedItems", +]); + +/** anyOf/oneOf whose branches are all `{ const: primitive }`: an enum (distinct values make oneOf the same). */ +function constUnion(branches: readonly unknown[], exactlyOne: boolean): Check | undefined { + const values: unknown[] = []; + for (const branch of branches) { + if (!isSchemaObject(branch) || !Object.hasOwn(branch, "const")) return undefined; + for (const k of Object.keys(branch)) if (k !== "const" && !NEUTRAL_KEYWORDS.has(k)) return undefined; + if (typeof branch.const === "object" && branch.const !== null) return undefined; + if (exactlyOne && values.includes(branch.const)) return undefined; + values.push(branch.const); + } + return enumCheck(values); +} + +// The tag of a tagged-union branch: a primitive const on a required property of an object schema. +function tagOf(branch: SchemaObject, key: string): unknown { + const properties = branch.properties; + if (branch.type !== "object" || !Array.isArray(branch.required) || !branch.required.includes(key)) return undefined; + const tag = isSchemaObject(properties) ? properties[key] : undefined; + if (!isSchemaObject(tag) || !Object.hasOwn(tag, "const")) return undefined; + return typeof tag.const === "object" && tag.const !== null ? undefined : tag.const; +} + +function number(value: unknown): number | undefined { + return typeof value === "number" ? value : undefined; +} + +function entries(value: unknown): [string, unknown][] { + return isSchemaObject(value) ? Object.entries(value) : []; +} diff --git a/packages/schema/src/explain.ts b/packages/schema/src/explain.ts new file mode 100644 index 000000000..3ab363986 --- /dev/null +++ b/packages/schema/src/explain.ts @@ -0,0 +1,390 @@ +// Errors of a failed value: a walk over the schema that reports each failing keyword at the value it failed on. +// It runs only for values that failed the check, using the compiler's checks for subschemas. + +import { newAnnotations } from "./annotations.ts"; +import { + codePoints, + constCheck, + containsCheck, + enumCheck, + isMultiple, + isObject, + NEVER, + TYPE_CHECKS, + uniqueCheck, +} from "./checks.ts"; +import type { Compiler } from "./compiler.ts"; +import { isSchemaObject } from "./resolver.ts"; +import { sharedRegex } from "./shared.ts"; +import type { SchemaObject, ValidationError } from "./types.ts"; + +/** Explains failures of one compiled schema. */ +export class Explainer { + private readonly c: Compiler; + private readonly schema: unknown; + /** + * The errors of `$ref` targets for object values, within one errors() call: a recursive union explains each branch + * to pick the one to blame, and every branch would explain the same children again, exponentially in the depth. + */ + private memo: Map> | undefined = undefined; + + constructor(compiler: Compiler, schema: unknown) { + this.c = compiler; + this.schema = schema; + compiler.root(); + } + + /** The errors of `value`, at least one (`value` failed the check). */ + errors(value: unknown): ValidationError[] { + const out: ValidationError[] = []; + try { + this.walk(this.schema, this.c.r.rootBase, value, "", out); + } finally { + this.memo = undefined; + } + if (out.length === 0) out.push({ path: "", keyword: "schema", message: "invalid" }); + return out; + } + + private ok(node: unknown, base: string, v: unknown): boolean { + return this.c.check(node, base)(v); + } + + // For $dynamicRef schemas this check runs outside the original dynamic scope, so their errors are approximate. + private walk(node: unknown, base: string, v: unknown, path: string, out: ValidationError[]): void { + if (node === true) return; + if (node === false) { + out.push({ path, keyword: "false", message: "no value is allowed here" }); + return; + } + if (!isSchemaObject(node)) return; + const b = this.c.r.baseFor(node, base); + if (this.ok(node, b, v)) return; + const start = out.length; + const push = (keyword: string, message: string, at = path) => { + out.push({ path: at, keyword, message }); + }; + if (this.c.draft === "7" && typeof node.$ref === "string") { + this.walkRef(node.$ref, b, v, path, out); + return; + } + if (node.type !== undefined) { + const types = Array.isArray(node.type) ? node.type : [node.type]; + const checks = types.map((t) => (typeof t === "string" ? TYPE_CHECKS.get(t) : undefined) ?? NEVER); + if (!checks.some((check) => check(v))) { + push("type", `expected ${types.join(" or ")}, got ${typeName(v)}`); + return; + } + } + if (Array.isArray(node.enum) && !enumCheck(node.enum)(v)) + push("enum", `expected one of ${listValues(node.enum)}`); + if (Object.hasOwn(node, "const") && !constCheck(node.const)(v)) + push("const", `expected ${JSON.stringify(node.const)}`); + if (typeof v === "string" && node.type !== "bytes") this.walkString(node, v, push); + if (v instanceof Uint8Array && node.type === "bytes") this.walkBytes(node, v, push); + if (typeof v === "number") this.walkNumber(node, v, push); + if (isObject(v)) this.walkObject(node, b, v, path, out, push); + if (Array.isArray(v)) this.walkArray(node, b, v, path, out, push); + if (Array.isArray(node.allOf)) for (const s of node.allOf) this.walk(s, b, v, path, out); + if (Array.isArray(node.anyOf)) this.walkUnion(node.anyOf, b, v, path, out, false); + if (Array.isArray(node.oneOf)) this.walkUnion(node.oneOf, b, v, path, out, true); + if (node.not !== undefined && this.ok(node.not, b, v)) push("not", "must not match the schema in not"); + if (node.if !== undefined) { + const branch = this.ok(node.if, b, v) ? node.then : node.else; + if (branch !== undefined) this.walk(branch, b, v, path, out); + } + if (typeof node.$ref === "string") this.walkRef(node.$ref, b, v, path, out); + if (out.length === start && (node.unevaluatedProperties !== undefined || node.unevaluatedItems !== undefined)) { + this.walkUnevaluated(node, b, v, path, out); + } + if (out.length === start && typeof node.$dynamicRef === "string") { + push("$dynamicRef", "does not match the dynamically referenced schema"); + } + } + + private walkRef(ref: string, base: string, v: unknown, path: string, out: ValidationError[]): void { + const target = this.c.r.resolve(ref, base); + if (target.node === false && this.c.r.isUnresolved(ref, base)) { + out.push({ path, keyword: "$ref", message: `unresolved reference ${ref}` }); + return; + } + if (!isSchemaObject(target.node) || typeof v !== "object" || v === null) { + this.walk(target.node, target.base, v, path, out); + return; + } + this.memo ??= new Map(); + let values = this.memo.get(target.node); + const known = values?.get(v); + if (known !== undefined && known.path === path) { + out.push(...known.errors); + return; + } + const errors: ValidationError[] = []; + this.walk(target.node, target.base, v, path, errors); + if (values === undefined) { + values = new Map(); + this.memo.set(target.node, values); + } + values.set(v, { path, errors }); + out.push(...errors); + } + + private walkString(node: SchemaObject, v: string, push: Push): void { + const n = codePoints(v); + if (typeof node.minLength === "number" && n < node.minLength) { + push("minLength", `expected at least ${node.minLength} characters`); + } + if (typeof node.maxLength === "number" && n > node.maxLength) { + push("maxLength", `expected at most ${node.maxLength} characters`); + } + if (typeof node.pattern === "string" && !sharedRegex(node.pattern).test(v)) { + push("pattern", `expected to match ${node.pattern}`); + } + const formats = this.c.formats; + const format = + formats !== undefined && typeof node.format === "string" && Object.hasOwn(formats, node.format) + ? formats[node.format] + : undefined; + if (format !== undefined && !format(v)) push("format", `expected format ${node.format}`); + } + + private walkBytes(node: SchemaObject, v: Uint8Array, push: Push): void { + if (typeof node.minLength === "number" && v.length < node.minLength) { + push("minLength", `expected at least ${node.minLength} bytes`); + } + if (typeof node.maxLength === "number" && v.length > node.maxLength) { + push("maxLength", `expected at most ${node.maxLength} bytes`); + } + } + + private walkNumber(node: SchemaObject, v: number, push: Push): void { + if (typeof node.minimum === "number" && v < node.minimum) push("minimum", `expected >= ${node.minimum}`); + if (typeof node.maximum === "number" && v > node.maximum) push("maximum", `expected <= ${node.maximum}`); + if (typeof node.exclusiveMinimum === "number" && v <= node.exclusiveMinimum) { + push("exclusiveMinimum", `expected > ${node.exclusiveMinimum}`); + } + if (typeof node.exclusiveMaximum === "number" && v >= node.exclusiveMaximum) { + push("exclusiveMaximum", `expected < ${node.exclusiveMaximum}`); + } + if (typeof node.multipleOf === "number" && !isMultiple(v, node.multipleOf)) { + push("multipleOf", `expected a multiple of ${node.multipleOf}`); + } + } + + private walkObject( + node: SchemaObject, + b: string, + v: Record, + path: string, + out: ValidationError[], + push: Push, + ): void { + const properties = isSchemaObject(node.properties) ? node.properties : {}; + const required = Array.isArray(node.required) ? node.required : []; + for (const k of required) { + if (typeof k === "string" && (!Object.hasOwn(v, k) || v[k] === undefined)) + push("required", "missing", pointer(path, k)); + } + for (const k of Object.keys(properties)) { + if (Object.hasOwn(v, k) && v[k] !== undefined) this.walk(properties[k], b, v[k], pointer(path, k), out); + } + const patterns = isSchemaObject(node.patternProperties) ? Object.entries(node.patternProperties) : []; + for (const k of Object.keys(v)) { + let matched = Object.hasOwn(properties, k); + for (const [p, s] of patterns) { + if (sharedRegex(p).test(k)) { + matched = true; + this.walk(s, b, v[k], pointer(path, k), out); + } + } + if (!matched && node.additionalProperties !== undefined) { + if (node.additionalProperties === false) push("additionalProperties", "unknown field", pointer(path, k)); + else this.walk(node.additionalProperties, b, v[k], pointer(path, k), out); + } + if (node.propertyNames !== undefined && !this.ok(node.propertyNames, b, k)) { + push("propertyNames", "invalid property name", pointer(path, k)); + } + } + const count = Object.keys(v).length; + if (typeof node.minProperties === "number" && count < node.minProperties) { + push("minProperties", `expected at least ${node.minProperties} properties`); + } + if (typeof node.maxProperties === "number" && count > node.maxProperties) { + push("maxProperties", `expected at most ${node.maxProperties} properties`); + } + const dependencies = { + ...(this.c.draft === "7" || !isSchemaObject(node.dependentRequired) ? {} : node.dependentRequired), + ...(isSchemaObject(node.dependencies) ? node.dependencies : {}), + }; + for (const [k, d] of Object.entries(dependencies)) { + if (!Object.hasOwn(v, k)) continue; + if (!Array.isArray(d)) this.walk(d, b, v, path, out); + else + for (const r of d) + if (!Object.hasOwn(v, r)) push("dependentRequired", `required when ${k} is present`, pointer(path, r)); + } + if (this.c.draft !== "7" && isSchemaObject(node.dependentSchemas)) { + for (const [k, s] of Object.entries(node.dependentSchemas)) + if (Object.hasOwn(v, k)) this.walk(s, b, v, path, out); + } + } + + private walkArray( + node: SchemaObject, + b: string, + v: unknown[], + path: string, + out: ValidationError[], + push: Push, + ): void { + let prefix: readonly unknown[] = []; + let rest: unknown; + if (Array.isArray(node.items)) { + prefix = node.items; + rest = node.additionalItems; + } else { + if (this.c.draft !== "7" && Array.isArray(node.prefixItems)) prefix = node.prefixItems; + rest = node.items; + } + for (let i = 0; i < v.length; i++) { + const s = i < prefix.length ? prefix[i] : rest; + if (s !== undefined) this.walk(s, b, v[i], `${path}/${i}`, out); + } + if (typeof node.minItems === "number" && v.length < node.minItems) { + push("minItems", `expected at least ${node.minItems} items`); + } + if (typeof node.maxItems === "number" && v.length > node.maxItems) { + push("maxItems", `expected at most ${node.maxItems} items`); + } + if (node.uniqueItems === true && !uniqueCheck(v)) push("uniqueItems", "expected unique items"); + if (node.contains !== undefined) { + const draft7 = this.c.draft === "7"; + const min = draft7 || typeof node.minContains !== "number" ? 1 : node.minContains; + const max = draft7 || typeof node.maxContains !== "number" ? Number.POSITIVE_INFINITY : node.maxContains; + if (!containsCheck(this.c.check(node.contains, b), min, max)(v)) { + push("contains", containsMessage(min, max)); + } + } + } + + private walkUnion( + branches: readonly unknown[], + b: string, + v: unknown, + path: string, + out: ValidationError[], + exactlyOne: boolean, + ): void { + const matches = branches.filter((s) => this.ok(s, b, v)).length; + if (matches === 1 || (!exactlyOne && matches > 0)) return; + const keyword = exactlyOne ? "oneOf" : "anyOf"; + if (matches > 1) { + out.push({ path, keyword, message: "matches more than one alternative" }); + return; + } + // A tagged union reports the branch its tag selects; otherwise the branch whose first error is deepest. + const tag = this.tagOf(branches, b); + if (tag !== undefined && isObject(v)) { + const i = tag.values.indexOf(v[tag.key]); + if (i < 0) + out.push({ + path: pointer(path, tag.key), + keyword, + message: `expected one of ${listValues(tag.values)}`, + }); + else this.walk(branches[i], b, v, path, out); + return; + } + // An object is blamed on the branch that declares most of its properties (the one it was most likely meant for), + // then on the branch whose first error is deepest. + let best: { errors: ValidationError[]; declared: number } | undefined; + for (const s of branches) { + const errors: ValidationError[] = []; + this.walk(s, b, v, path, errors); + if (errors.length === 0) continue; + const declared = isObject(v) ? this.declaredKeys(s, b, v) : 0; + const better = + best === undefined || + declared > best.declared || + (declared === best.declared && errors[0].path.length > best.errors[0].path.length); + if (better) best = { errors, declared }; + } + out.push(...(best?.errors ?? [{ path, keyword, message: "matches no alternative" }])); + } + + /** How many of the keys of `v` the branch `s` (through a `$ref`) declares in `properties`. */ + private declaredKeys(s: unknown, b: string, v: Record): number { + const node = isSchemaObject(s) && typeof s.$ref === "string" ? this.c.r.resolve(s.$ref, b).node : s; + const properties = isSchemaObject(node) && isSchemaObject(node.properties) ? node.properties : {}; + return Object.keys(v).filter((k) => Object.hasOwn(properties, k)).length; + } + + private tagOf(branches: readonly unknown[], b: string): { key: string; values: unknown[] } | undefined { + const resolved = branches.map((s) => + isSchemaObject(s) && typeof s.$ref === "string" ? this.c.r.resolve(s.$ref, b).node : s, + ); + const first = resolved[0]; + if (!isSchemaObject(first) || !isSchemaObject(first.properties)) return undefined; + for (const key of Object.keys(first.properties)) { + const values = resolved.map((s) => { + const tag = isSchemaObject(s) && isSchemaObject(s.properties) ? s.properties[key] : undefined; + return isSchemaObject(tag) ? tag.const : undefined; + }); + const primitive = values.every((x) => x !== undefined && (typeof x !== "object" || x === null)); + if (primitive && new Set(values).size === values.length) return { key, values }; + } + return undefined; + } + + private walkUnevaluated(node: SchemaObject, b: string, v: unknown, path: string, out: ValidationError[]): void { + const { unevaluatedProperties, unevaluatedItems, ...rest } = node; + const a = newAnnotations(); + this.c.annotating(rest, b)(v, a); + if (isObject(v) && unevaluatedProperties !== undefined && !a.allProps) { + for (const k of Object.keys(v)) { + if (a.props?.has(k)) continue; + if (unevaluatedProperties === false) { + out.push({ path: pointer(path, k), keyword: "unevaluatedProperties", message: "unknown field" }); + } else this.walk(unevaluatedProperties, b, v[k], pointer(path, k), out); + } + } + if (Array.isArray(v) && unevaluatedItems !== undefined && !a.allItems) { + for (let i = a.items; i < v.length; i++) { + if (a.indexes?.has(i)) continue; + if (unevaluatedItems === false) { + out.push({ path: `${path}/${i}`, keyword: "unevaluatedItems", message: "unexpected item" }); + } else this.walk(unevaluatedItems, b, v[i], `${path}/${i}`, out); + } + } + } +} + +type Push = (keyword: string, message: string, at?: string) => void; + +// Enum errors name the allowed values; a long list (an enum of thousands) is cut, so the message stays readable. +const LISTED_VALUES = 20; + +/** `values` as JSON, at most LISTED_VALUES of them, then how many more there are. */ +function listValues(values: readonly unknown[]): string { + if (values.length <= LISTED_VALUES) return JSON.stringify(values); + const listed = JSON.stringify(values.slice(0, LISTED_VALUES)); + return `${listed.slice(0, -1)}, ... and ${values.length - LISTED_VALUES} more]`; +} + +function containsMessage(min: number, max: number): string { + if (max === Number.POSITIVE_INFINITY) return `expected at least ${min} matching ${min === 1 ? "item" : "items"}`; + if (min === 0) return `expected at most ${max} matching ${max === 1 ? "item" : "items"}`; + return `expected between ${min} and ${max} matching items`; +} + +function pointer(path: string, key: string | number): string { + return `${path}/${String(key).replace(/~/g, "~0").replace(/\//g, "~1")}`; +} + +function typeName(v: unknown): string { + if (v === null) return "null"; + if (Array.isArray(v)) return "array"; + if (v instanceof Uint8Array) return "bytes"; + if (typeof v === "number" && Number.isInteger(v)) return "integer"; + return typeof v; +} diff --git a/packages/schema/src/formats/index.ts b/packages/schema/src/formats/index.ts new file mode 100644 index 000000000..05d8824ed --- /dev/null +++ b/packages/schema/src/formats/index.ts @@ -0,0 +1,53 @@ +// The format checks of JSON Schema 2020-12, for compile()'s `formats` option. An optional entry point: schemas +// compiled without it treat `format` as an annotation and do not bundle these checks. + +import type { FormatCheck } from "../types.ts"; +import { IsDate } from "./typebox/date.ts"; +import { IsDateTime } from "./typebox/date_time.ts"; +import { IsDuration } from "./typebox/duration.ts"; +import { IsEmail } from "./typebox/email.ts"; +import { IsHostname } from "./typebox/hostname.ts"; +import { IsIdnEmail } from "./typebox/idn_email.ts"; +import { IsIdnHostname } from "./typebox/idn_hostname.ts"; +import { IsIPv4 } from "./typebox/ipv4.ts"; +import { IsIPv6 } from "./typebox/ipv6.ts"; +import { IsIri } from "./typebox/iri.ts"; +import { IsIriReference } from "./typebox/iri_reference.ts"; +import { IsJsonPointer } from "./typebox/json_pointer.ts"; +import { IsJsonPointerUriFragment } from "./typebox/json_pointer_uri_fragment.ts"; +import { IsRegex } from "./typebox/regex.ts"; +import { IsRelativeJsonPointer } from "./typebox/relative_json_pointer.ts"; +import { IsTime } from "./typebox/time.ts"; +import { IsUri } from "./typebox/uri.ts"; +import { IsUriReference } from "./typebox/uri_reference.ts"; +import { IsUriTemplate } from "./typebox/uri_template.ts"; +import { IsUrl } from "./typebox/url.ts"; +import { IsUuid } from "./typebox/uuid.ts"; + +/** + * Every format JSON Schema 2020-12 defines, by name, plus `url` and `json-pointer-uri-fragment`, which TypeBox and + * ajv-formats also check. The checks are adapted from TypeBox 1.3.27 (MIT). + */ +export const formats: Readonly> = Object.freeze({ + date: IsDate, + "date-time": IsDateTime, + duration: IsDuration, + email: IsEmail, + hostname: IsHostname, + "idn-email": IsIdnEmail, + "idn-hostname": IsIdnHostname, + ipv4: IsIPv4, + ipv6: IsIPv6, + iri: IsIri, + "iri-reference": IsIriReference, + "json-pointer": IsJsonPointer, + "json-pointer-uri-fragment": IsJsonPointerUriFragment, + regex: IsRegex, + "relative-json-pointer": IsRelativeJsonPointer, + time: IsTime, + uri: IsUri, + "uri-reference": IsUriReference, + "uri-template": IsUriTemplate, + url: IsUrl, + uuid: IsUuid, +}); diff --git a/packages/schema/src/formats/typebox/date.ts b/packages/schema/src/formats/typebox/date.ts new file mode 100644 index 000000000..e35cf0770 --- /dev/null +++ b/packages/schema/src/formats/typebox/date.ts @@ -0,0 +1,47 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const DAYS = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; +const DATE = /^(\d\d\d\d)-(\d\d)-(\d\d)$/; + +function IsLeapYear(year: number): boolean { + return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0); +} +/** + * Returns true if the value is a ISO8601 Date component string + * @description https://datatracker.ietf.org/doc/html/rfc3339 + * @example `2020-12-12` + */ +export function IsDate(value: string): boolean { + const matches: string[] | null = DATE.exec(value); + if (!matches) return false; + const year: number = +matches[1]; + const month: number = +matches[2]; + const day: number = +matches[3]; + return month >= 1 && month <= 12 && day >= 1 && day <= (month === 2 && IsLeapYear(year) ? 29 : DAYS[month]); +} diff --git a/packages/schema/src/formats/typebox/date_time.ts b/packages/schema/src/formats/typebox/date_time.ts new file mode 100644 index 000000000..df4174a40 --- /dev/null +++ b/packages/schema/src/formats/typebox/date_time.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import { IsDate } from "./date.ts"; +import { IsTime } from "./time.ts"; + +/** + * Returns true if the value is a ISO8601 DateTime string + * @specification https://datatracker.ietf.org/doc/html/rfc3339 + * @example `2020-12-12T20:20:40+00:00` + */ +export function IsDateTime(value: string): boolean { + const dateTime: string[] = value.split(/T/i); + return dateTime.length === 2 && IsDate(dateTime[0]) && IsTime(dateTime[1]); +} diff --git a/packages/schema/src/formats/typebox/duration.ts b/packages/schema/src/formats/typebox/duration.ts new file mode 100644 index 000000000..20d812c4f --- /dev/null +++ b/packages/schema/src/formats/typebox/duration.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Duration = + /^P((\d+Y(\d+M(\d+D)?)?|\d+M(\d+D)?|\d+D)(T(\d+H(\d+M(\d+S)?)?|\d+M(\d+S)?|\d+S))?|T(\d+H(\d+M(\d+S)?)?|\d+M(\d+S)?|\d+S)|\d+W)$/; + +/** + * Returns true if the value is a valid ISO-8601 duration. + * @specification https://tools.ietf.org/html/rfc3339 + */ +export function IsDuration(value: string): boolean { + return Duration.test(value); +} diff --git a/packages/schema/src/formats/typebox/email.ts b/packages/schema/src/formats/typebox/email.ts new file mode 100644 index 000000000..a6840daed --- /dev/null +++ b/packages/schema/src/formats/typebox/email.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Email = + /^(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[^"\\]|\\[\x20-\x7e])*")@(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)*|\[(?:IPv6:[a-f0-9:]+|(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(?:\.(?:25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3})\])$/i; + +/** + * Returns true if the value is an Email + * @specification https://datatracker.ietf.org/doc/html/rfc5322 + */ +export function IsEmail(value: string): boolean { + return Email.test(value); +} diff --git a/packages/schema/src/formats/typebox/hostname.ts b/packages/schema/src/formats/typebox/hostname.ts new file mode 100644 index 000000000..68c3710fb --- /dev/null +++ b/packages/schema/src/formats/typebox/hostname.ts @@ -0,0 +1,39 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Idna from "./idna/index.ts"; + +/** + * Returns true if the value is a valid hostname. + * @specification https://tools.ietf.org/html/rfc1123 + * @specification https://tools.ietf.org/html/rfc5891 + * @specification https://tools.ietf.org/html/rfc5892 + */ +export function IsHostname(value: string): boolean { + return Idna.IsHostname(value); +} diff --git a/packages/schema/src/formats/typebox/idn_email.ts b/packages/schema/src/formats/typebox/idn_email.ts new file mode 100644 index 000000000..92e3abdaf --- /dev/null +++ b/packages/schema/src/formats/typebox/idn_email.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IdnEmail = + /^(?:[A-Za-z0-9!#$%&'*+/=?^_`{|}~\u{0080}-\u{10FFFF}-]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~\u{0080}-\u{10FFFF}-]+)*|"(?:[^"\\]|\\.)*")@[\p{L}\p{N}](?:[\p{L}\p{N}-]{0,62})(?> 1; + delta += Math.floor(delta / numPoints); + let k = 0; + while (delta > ((PUNYCODE_BASE - PUNYCODE_TMIN) * PUNYCODE_TMAX) >> 1) { + delta = Math.floor(delta / (PUNYCODE_BASE - PUNYCODE_TMIN)); + k += PUNYCODE_BASE; + } + return k + Math.floor(((PUNYCODE_BASE - PUNYCODE_TMIN + 1) * delta) / (delta + PUNYCODE_SKEW)); +} +// ------------------------------------------------------------------ +// Decode +// ------------------------------------------------------------------ +export function Decode(value: string): string { + const output: number[] = []; + let n = PUNYCODE_INITIAL_N; + let i = 0; + let bias = PUNYCODE_INITIAL_BIAS; + const delimIdx = value.lastIndexOf("-"); + if (delimIdx > 0) { + for (let j = 0; j < delimIdx; j++) { + const cp = value.charCodeAt(j); + if (cp >= 128) ThrowDontCare(); // throw new Error('Invalid punycode: non-basic before delimiter') + output.push(cp); + } + } + let inIdx = delimIdx < 0 ? 0 : delimIdx + 1; + while (inIdx < value.length) { + const oldi = i; + let w = 1; + let k = PUNYCODE_BASE; + while (true) { + if (inIdx >= value.length) ThrowDontCare(); // throw new Error('Invalid punycode: unexpected end of input') + const ch = value.charCodeAt(inIdx++); + let digit: number; + if (ch >= 0x61 && ch <= 0x7a) + digit = ch - 0x61; // a-z => 0-25 + else if (ch >= 0x30 && ch <= 0x39) + digit = ch - 0x30 + 26; // 0-9 => 26-35 + // else if (ch >= 0x41 && ch <= 0x5a) digit = ch - 0x41 // A-Z => 0-25 // (no-spec-coverage) + else ThrowDontCare(); // throw new Error('Invalid punycode: bad digit character') + i += digit * w; + const t = k <= bias ? PUNYCODE_TMIN : k >= bias + PUNYCODE_TMAX ? PUNYCODE_TMAX : k - bias; + if (digit < t) break; + w *= PUNYCODE_BASE - t; + k += PUNYCODE_BASE; + } + const outLen = output.length + 1; + bias = Adapt(i - oldi, outLen, oldi === 0); + n += Math.floor(i / outLen); + i %= outLen; + output.splice(i, 0, n); + i++; + } + return String.fromCodePoint(...output); +} +// ------------------------------------------------------------------ +// DigitToChar +// ------------------------------------------------------------------ +function DigitToChar(digit: number): string { + return digit < 26 ? String.fromCharCode(digit + 0x61) : String.fromCharCode(digit - 26 + 0x30); +} +// ------------------------------------------------------------------ +// Encode +// +// Encodes a Unicode string into its Punycode payload (the part that +// follows the 'xn--' ACE prefix). Used to compute the true A-label +// length of a U-label, since the 63-octet label length limit (RFC +// 5891 §4.2.4) applies to the encoded form, not the code point count. +// ------------------------------------------------------------------ +const RE_REPLACE_NON_ASCII = new RegExp(Pattern.RE_NON_ASCII.source, "gu"); +export function Encode(input: string): string { + // Extract basic ASCII characters and set initial output array state + const basic = input.replace(RE_REPLACE_NON_ASCII, ""); + const basicLength = basic.length; + const result = basicLength > 0 ? [basic, "-"] : []; + // Extract unicode codepoints + const codePoints = Array.from(input, (char) => char.codePointAt(0)!); + let n = PUNYCODE_INITIAL_N; + let delta = 0; + let bias = PUNYCODE_INITIAL_BIAS; + let handledCPCount = basicLength; + while (handledCPCount < codePoints.length) { + let m = Infinity; + for (const cp of codePoints) { + if (cp >= n && cp < m) m = cp; + } + delta += (m - n) * (handledCPCount + 1); + n = m; + for (const cp of codePoints) { + if (cp < n) delta++; + if (cp === n) { + let q = delta; + for (let k = PUNYCODE_BASE; ; k += PUNYCODE_BASE) { + const t = k <= bias ? PUNYCODE_TMIN : k >= bias + PUNYCODE_TMAX ? PUNYCODE_TMAX : k - bias; + if (q < t) break; + const digit = t + ((q - t) % (PUNYCODE_BASE - t)); + result.push(DigitToChar(digit)); + q = Math.floor((q - t) / (PUNYCODE_BASE - t)); + } + result.push(DigitToChar(q)); + bias = Adapt(delta, handledCPCount + 1, handledCPCount === basicLength); + delta = 0; + handledCPCount++; + } + } + delta++; + n++; + } + return result.join(""); +} diff --git a/packages/schema/src/formats/typebox/idna/hostname.ts b/packages/schema/src/formats/typebox/idna/hostname.ts new file mode 100644 index 000000000..ade2322b4 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/hostname.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as LabelAscii from "./label/ascii.ts"; +import * as LabelPuny from "./label/puny.ts"; + +function IsValidLabelLength(value: string): boolean { + return value.length > 0 && value.length <= 63; +} +function IsLabel(value: string): boolean { + return IsValidLabelLength(value) && (LabelPuny.IsPunyLabel(value) || LabelAscii.IsAsciiLabel(value)); +} +export function IsHostname(value: string): boolean { + if (value.length === 0 || value.length > 253) return false; + if (value.charCodeAt(value.length - 1) === 46) return false; + return value.split(".").every((label) => IsLabel(label)); +} diff --git a/packages/schema/src/formats/typebox/idna/idn-hostname.ts b/packages/schema/src/formats/typebox/idna/idn-hostname.ts new file mode 100644 index 000000000..d4517faac --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/idn-hostname.ts @@ -0,0 +1,72 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatBidi from "./format/bidi.ts"; +import * as LabelPuny from "./label/puny.ts"; +import * as LabelUnicode from "./label/unicode.ts"; + +// ------------------------------------------------------------------ +// IsLabel +// ------------------------------------------------------------------ +function IsValidLabelLength(value: string): boolean { + return value.length > 0 && value.length <= 63; +} +function IsLabel(value: string): boolean { + return IsValidLabelLength(value) && (LabelPuny.IsPunyLabel(value) || LabelUnicode.IsUnicodeLabel(value)); +} +// ------------------------------------------------------------------ +// NormalizeHostname +// +// Normalizes a hostname for label validation. Maps fullwidth +// codepoints to their ASCII equivalent (e.g. 'ａ' U+FF41 maps to +// 'a'), then normalizes for NFC such that equivalent codepoint +// sequences can be uniformly compared. It also removes codepoints +// that IDNA mapping ignores, and converts full stop variants to +// ASCII '.' so the hostname can be split into labels. +// ------------------------------------------------------------------ +function NormalizeHostname(value: string): string { + return ( + value + .replace(/[\uff01-\uff5e]/g, (char) => String.fromCharCode(char.charCodeAt(0) - 0xfee0)) // RE_FULLWIDTH_MAPPING + .normalize("NFC") + // biome-ignore lint/suspicious/noMisleadingCharacterClass: a set of single code points to remove (u flag), by design. + .replace(/[\u00ad\u034f\u180b-\u180d\u200b\ufe00-\ufe0f\u{e0100}-\u{e01ef}]/gu, "") // RE_IGNORED_MAPPING + .replace(/[\u002E\u3002\uFF0E\uFF61]/g, ".") + ); // RE_FULL_STOP_MAPPING +} +// ------------------------------------------------------------------ +// IsIdnHostname +// ------------------------------------------------------------------ +export function IsIdnHostname(value: string): boolean { + if (value.length === 0 || value.includes(" ")) return false; + const normalized = NormalizeHostname(value); + if (normalized.length > 253) return false; + const labels = normalized.split("."); + const hasBidiChars = labels.some((label) => FormatBidi.HasBidiChars(label)); + return labels.every((label) => IsLabel(label) && (!hasBidiChars || FormatBidi.SatisfiesBidiRule(label))); +} diff --git a/packages/schema/src/formats/typebox/idna/index.ts b/packages/schema/src/formats/typebox/idna/index.ts new file mode 100644 index 000000000..c5d10175e --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/index.ts @@ -0,0 +1,30 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +export * from "./hostname.ts"; +export * from "./idn-hostname.ts"; diff --git a/packages/schema/src/formats/typebox/idna/label/ascii.ts b/packages/schema/src/formats/typebox/idna/label/ascii.ts new file mode 100644 index 000000000..704e8f944 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/ascii.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as Pattern from "../pattern/pattern.ts"; + +// ------------------------------------------------------------------- +// Validates an ASCII LDH label per RFC 5891 §4.2.3.1 +// ------------------------------------------------------------------- +export function IsAsciiLabel(value: string): boolean { + return ( + Pattern.RE_RULE_HYPHEN_PLACEMENT.test(value) && + Pattern.RE_RULE_NOT_RESERVED_ACE.test(value) && + Pattern.RE_ASCII_LDH.test(value) + ); +} diff --git a/packages/schema/src/formats/typebox/idna/label/puny.ts b/packages/schema/src/formats/typebox/idna/label/puny.ts new file mode 100644 index 000000000..23b288ce2 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/puny.ts @@ -0,0 +1,53 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatPuny from "../format/puny.ts"; +import * as Pattern from "../pattern/pattern.ts"; +import * as LabelUnicode from "./unicode.ts"; + +// ------------------------------------------------------------------ +// IsPunyLabel +// ------------------------------------------------------------------ +export function IsPunyLabel(value: string): boolean { + if (!FormatPuny.IsAcePrefixed(value)) return false; + try { + const body = value.slice(4).toLowerCase(); + // A payload consisting of only a hyphen, or starting with the delimiter + // (like "-9uc"), has 0 basic ASCII characters preceding it and is malformed. + if (body.lastIndexOf("-") === 0) return false; + const decoded = FormatPuny.Decode(body); + // RFC 5890 §2.3.2.1 - a U-label must contain at least one non-ASCII + // character. Code-unit level check: any non-ASCII codepoint has at least + // one UTF-16 unit >= 0x80 (surrogate halves included), so no need to + // decode codepoints or allocate a character array to answer this. + if (!Pattern.RE_NON_ASCII.test(decoded)) return false; + return LabelUnicode.IsUnicodeLabel(decoded); + } catch { + return false; + } +} diff --git a/packages/schema/src/formats/typebox/idna/label/unicode.ts b/packages/schema/src/formats/typebox/idna/label/unicode.ts new file mode 100644 index 000000000..615942427 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/label/unicode.ts @@ -0,0 +1,122 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +import * as FormatBidi from "../format/bidi.ts"; +import * as FormatPuny from "../format/puny.ts"; +import * as Pattern from "../pattern/pattern.ts"; + +// ------------------------------------------------------------------ +// ExceedsMaxALabelLength +// +// RFC 5890 §2.3.2.1: an A-label (the ACE-encoded form of a label, +// "xn--" + punycode) must not exceed 63 octets. Only labels +// containing non-ASCII characters need checking - a pure-ASCII label +// has no "xn--" prefix and is used as-is, so its punycode-encoded +// length is irrelevant. The ASCII check is a cheap short-circuit that +// avoids running the punycode encoder on labels that don't need it. +// ------------------------------------------------------------------ +function ExceedsMaxALabelLength(value: string): boolean { + return Pattern.RE_NON_ASCII.test(value) && FormatPuny.Encode(value).length + 4 > 63; +} +// ------------------------------------------------------------------ +// HasInvalidHyphens +// +// RFC 5891 §4.2.3.2: label must not start or end with a hyphen, and +// must not have hyphens in the 3rd and 4th codepoint positions (the +// ACE prefix reservation, e.g. "xn--"). Operates on `chars` (the +// per-codepoint string array) rather than raw string indices so +// supplementary-plane characters earlier in the label don't throw +// off the codepoint-position count. +// ------------------------------------------------------------------ +function HasInvalidHyphens(chars: string[]): boolean { + if (chars[0] === "-" || chars[chars.length - 1] === "-") return true; + return chars.slice(2).join("").startsWith("--"); +} +// ------------------------------------------------------------------ +// IsUnicodeLabel +// ------------------------------------------------------------------ +export function IsUnicodeLabel(value: string): boolean { + // RFC 5890 §2.3.2.1: Ensure Max Length + if (ExceedsMaxALabelLength(value)) return false; + // RFC 5893 - Bidi Rule (applies whenever this label itself is RTL) + if (FormatBidi.HasRightToLeftCharacters(value) && !FormatBidi.SatisfiesBidiRule(value)) return false; + const chars = [...value]; + const codePoints = chars.map((c) => c.codePointAt(0)!); + const length = codePoints.length; + // RFC 5891 §4.2.3.2: Hyphen rules + if (HasInvalidHyphens(chars)) return false; + // RFC 5891 §4.2.3.2 - Must not begin with a combining mark + if (Pattern.RE_COMBINING_MARK.test(chars[0])) return false; + let hasJapanese = false; + // let hasArabicIndic = false // (no-spec-coverage) + // let hasExtendedArabicIndic = false // (no-spec-coverage) + for (let i = 0; i < length; i++) { + const codePoint = codePoints[i]; + const char = chars[i]; + // 1. DISALLOWED exceptions + if (Pattern.RE_RFC5892_DISALLOWED.test(char)) return false; + // 1b. General category (PVALID) enforcement + if (!Pattern.RE_PERMITTED_CATEGORY.test(char)) return false; + // 2. Collect Flags + if (Pattern.RE_SCRIPT_JAPANESE.test(char)) hasJapanese = true; + // if (Pattern.RE_ARABIC_INDIC_DIGIT.test(char)) hasArabicIndic = true // (no-spec-coverage) + // if (Pattern.RE_EXT_ARABIC_INDIC_DIGIT.test(char)) hasExtendedArabicIndic = true // (no-spec-coverage) + // 3. CONTEXTO / CONTEXTJ Neighbor Rules + const prev = codePoints[i - 1], + next = codePoints[i + 1]; + switch (codePoint) { + case 0x00b7: + if (prev !== 0x006c || next !== 0x006c) return false; + break; // MIDDLE DOT (Catalan) + case 0x0375: + if (!next || !Pattern.RE_SCRIPT_GREEK.test(chars[i + 1])) return false; + break; // Greek KERAIA + case 0x05f3: + case 0x05f4: + if (!prev || !Pattern.RE_SCRIPT_HEBREW.test(chars[i - 1])) return false; + break; // Hebrew GERESH + case 0x200c: // ZWNJ - RFC 5892 Appendix A.1 + // Note: Full validation requires a Unicode joining type table. We reject + // only when preceded by ASCII (U+0000-U+007F), which can never satisfy + // the Virama or cursive-joining rules. + if (!prev || (prev < 0x0080 && !Pattern.RE_VIRAMA.test(chars[i - 1]))) return false; + break; + case 0x200d: // ZWJ: RFC 5892 Appendix A.2 + if (!prev || !Pattern.RE_VIRAMA.test(chars[i - 1])) return false; + break; + case 0x30fb: // Checked at end via hasJapanese + break; // KATAKANA MIDDLE DOT + } + } + // 4. Global Context Validations (Post-loop) + // RFC 5892 Appendix A.7 - Katakana Middle Dot requirement + if (value.includes("\u30fb") && !hasJapanese) return false; + // RFC 5892 Appendix A.8/A.9 - Mixing Arabic Digits + // if (hasArabicIndic && hasExtendedArabicIndic) return false // (no-spec-coverage) + return true; +} diff --git a/packages/schema/src/formats/typebox/idna/pattern/pattern.ts b/packages/schema/src/formats/typebox/idna/pattern/pattern.ts new file mode 100644 index 000000000..dfb55c1d6 --- /dev/null +++ b/packages/schema/src/formats/typebox/idna/pattern/pattern.ts @@ -0,0 +1,116 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-fmt-ignore-file + +// ------------------------------------------------------------------- +// Validation Rules +// ------------------------------------------------------------------- +export const RE_RULE_HYPHEN_PLACEMENT = /^(?!-).*(? regexp.source).join("|"), + "u", +); + +export const RE_PERMITTED_CATEGORY = new RegExp( + [ + RE_LETTER, + RE_EUROPEAN_SEPARATOR, + RE_COMMON_SEPARATOR, + RE_NUMBER_DECIMAL, + RE_MARK_NONSPACING, + RE_MARK_SPACING_COMBINING, + RE_CONTEXTO_EXCEPTIONS, + RE_PVALID_EXCEPTIONS, + ] + .map((regexp) => regexp.source) + .join("|"), + "u", +); diff --git a/packages/schema/src/formats/typebox/ipv4.ts b/packages/schema/src/formats/typebox/ipv4.ts new file mode 100644 index 000000000..2c6bf3c7b --- /dev/null +++ b/packages/schema/src/formats/typebox/ipv4.ts @@ -0,0 +1,37 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IPv4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; + +/** + * Returns true if the value is a IPV4 address + * @specification http://tools.ietf.org/html/rfc2673#section-3.2 + */ +export function IsIPv4(value: string): boolean { + return IPv4.test(value); +} diff --git a/packages/schema/src/formats/typebox/ipv6.ts b/packages/schema/src/formats/typebox/ipv6.ts new file mode 100644 index 000000000..4cc46deac --- /dev/null +++ b/packages/schema/src/formats/typebox/ipv6.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const IPv6 = + /^(?:(?:(?:[0-9a-f]{1,4}:){6}|::(?:[0-9a-f]{1,4}:){5}|(?:[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){4}|(?:(?:[0-9a-f]{1,4}:)?[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){3}|(?:(?:[0-9a-f]{1,4}:){0,2}[0-9a-f]{1,4})?::(?:[0-9a-f]{1,4}:){2}|(?:(?:[0-9a-f]{1,4}:){0,3}[0-9a-f]{1,4})?::[0-9a-f]{1,4}:|(?:(?:[0-9a-f]{1,4}:){0,4}[0-9a-f]{1,4})?::)(?:[0-9a-f]{1,4}:[0-9a-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[0-9a-f]{1,4}:){0,5}[0-9a-f]{1,4})?::[0-9a-f]{1,4}|(?:(?:[0-9a-f]{1,4}:){0,6}[0-9a-f]{1,4})?::)$/i; + +/** + * Returns true if the value is an IPv6 address + * @specification http://tools.ietf.org/html/rfc2373#section-2.2 + */ +export function IsIPv6(value: string): boolean { + return IPv6.test(value); +} diff --git a/packages/schema/src/formats/typebox/iri.ts b/packages/schema/src/formats/typebox/iri.ts new file mode 100644 index 000000000..8ee1d0540 --- /dev/null +++ b/packages/schema/src/formats/typebox/iri.ts @@ -0,0 +1,63 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +const IpvFutureMatchMaxLength = 2048; +const IpvFutureMatch = /\[[vV][0-9a-fA-F]+\.[^\]]+\]/; // Guarded By IpvFutureMatchMaxLength +const InvalidIriChars = /[\x00-\x20<>^`{|}\\]/; +const InvalidPercentEncoding = /%(?![0-9a-fA-F]{2})/; + +// ------------------------------------------------------------------ +// NarrowIpvFuture +// +// Substitutes an IPvFuture address with a standard IPv6 loopback +// address ([::1]). We do this because the native URL.canParse +// (WHATWG standard) rejects IPvFuture literals, which are +// otherwise valid in RFC 3987. Because regex substitution can +// be expensive on large strings, this operation is strictly +// limited to inputs under a defined length threshold. +// +// (review-optimization) +// +// ------------------------------------------------------------------ +function NarrowIpvFuture(value: string): string { + return value.length < IpvFutureMatchMaxLength ? value.replace(IpvFutureMatch, "[::1]") : value; +} +/** + * Returns true if the value is a valid Internationalized Resource Identifier. + * @specification https://datatracker.ietf.org/doc/html/rfc3987 + */ +export function IsIri(value: string): boolean { + // 1. Reject strings containing unencoded whitespace or illegal control characters. + if (InvalidIriChars.test(value)) return false; + // 2. Reject malformed percent-encoding triplets. + if (InvalidPercentEncoding.test(value)) return false; + // 3. Delegate to the native URL parser, patching the IPvFuture edge case beforehand. + return URL.canParse(NarrowIpvFuture(value)); +} diff --git a/packages/schema/src/formats/typebox/iri_reference.ts b/packages/schema/src/formats/typebox/iri_reference.ts new file mode 100644 index 000000000..5e45c75ee --- /dev/null +++ b/packages/schema/src/formats/typebox/iri_reference.ts @@ -0,0 +1,42 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +// Rejects unencoded spaces (0x20), backslashes, control characters (0x00-0x1F, 0x7F), or malformed percent-encodings +const InvalidIriChars = /[\x00-\x20\x7F\\]|%(?![0-9a-fA-F]{2})/; +// Detects malformed scheme/authority patterns lacking a colon (e.g. "httpx//example.com") +const MalformedScheme = /^[a-zA-Z][a-zA-Z0-9+\-.]*\/\//; + +/** + * Returns true if the value is an IRI reference + * @specification https://tools.ietf.org/html/rfc3987 + */ +export function IsIriReference(value: string): boolean { + return !InvalidIriChars.test(value) && !MalformedScheme.test(value) && URL.canParse(value, "http://example.com"); +} diff --git a/packages/schema/src/formats/typebox/json_pointer.ts b/packages/schema/src/formats/typebox/json_pointer.ts new file mode 100644 index 000000000..c2d171aef --- /dev/null +++ b/packages/schema/src/formats/typebox/json_pointer.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const JsonPointer = /^(?:\/(?:[^~/]|~0|~1)*)*$/; + +/** + * Returns true if the value is a json pointer + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsJsonPointer(value: string): boolean { + return JsonPointer.test(value); +} diff --git a/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts b/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts new file mode 100644 index 000000000..cac809cef --- /dev/null +++ b/packages/schema/src/formats/typebox/json_pointer_uri_fragment.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const JsonPointerUriFragment = /^#(?:\/(?:[a-z0-9_\-.!$&'()*+,;:=@]|%[0-9a-f]{2}|~0|~1)*)*$/i; + +/** + * Returns true if the value is a json pointer uri fragment + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsJsonPointerUriFragment(value: string): boolean { + return JsonPointerUriFragment.test(value); +} diff --git a/packages/schema/src/formats/typebox/regex.ts b/packages/schema/src/formats/typebox/regex.ts new file mode 100644 index 000000000..52cff8d2a --- /dev/null +++ b/packages/schema/src/formats/typebox/regex.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +/** + * Returns true if the value is a regular expression string pattern + * @specification https://ecma-international.org/ecma-262 + */ +export function IsRegex(value: string): boolean { + try { + new RegExp(value, "u"); + return true; + } catch { + return false; + } +} diff --git a/packages/schema/src/formats/typebox/relative_json_pointer.ts b/packages/schema/src/formats/typebox/relative_json_pointer.ts new file mode 100644 index 000000000..f2dfd6fbd --- /dev/null +++ b/packages/schema/src/formats/typebox/relative_json_pointer.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const RelativeJsonPointer = /^(?:0|[1-9][0-9]*)(?:#|(?:\/(?:[^~/]|~0|~1)*)*)$/; + +/** + * Returns true if the value is a relative json pointer + * @specification https://datatracker.ietf.org/doc/html/rfc6901 + * @source https://github.com/ajv-validator/ajv-formats + */ +export function IsRelativeJsonPointer(value: string): boolean { + return RelativeJsonPointer.test(value); +} diff --git a/packages/schema/src/formats/typebox/time.ts b/packages/schema/src/formats/typebox/time.ts new file mode 100644 index 000000000..bbc9c6113 --- /dev/null +++ b/packages/schema/src/formats/typebox/time.ts @@ -0,0 +1,56 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const TIME = /^(\d\d):(\d\d):(\d\d)(?:\.\d+)?(?:([Zz])|([+-])(\d\d):(\d\d))?$/; + +/** + * Returns true if the value is an ISO time string + * @specification https://datatracker.ietf.org/doc/html/rfc3339 + */ +export function IsTime(value: string, strictTimeZone: boolean = true): boolean { + const matches = TIME.exec(value); + if (!matches) return false; + // Require timezone offset or 'Z'/'z' when strictTimeZone is true + if (strictTimeZone && !matches[4] && !matches[5]) return false; + const hr = +matches[1]; + const min = +matches[2]; + const sec = +matches[3]; + if (hr > 23 || min > 59 || sec > 60) return false; + if (matches[5]) { + const tzH = +matches[6]; + const tzM = +matches[7]; + if (tzH > 23 || tzM > 59) return false; + } + if (sec < 60) return true; + // Leap second handling: must normalize to 23:59:60 UTC (1439 total UTC minutes) + const tzSign = matches[5] === "-" ? -1 : 1; + const tzH = +(matches[6] || 0); + const tzM = +(matches[7] || 0); + const totalUtcMin = hr * 60 + min - tzSign * (tzH * 60 + tzM); + return ((totalUtcMin % 1440) + 1440) % 1440 === 1439; +} diff --git a/packages/schema/src/formats/typebox/uri.ts b/packages/schema/src/formats/typebox/uri.ts new file mode 100644 index 000000000..5aadeab17 --- /dev/null +++ b/packages/schema/src/formats/typebox/uri.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Uri = + /^[a-z][a-z0-9+\-.]*:(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?(?:\?(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + +/** + * Returns true if the value is a valid Uniform Resource Identifier. + * @specification https://tools.ietf.org/html/rfc3986 + */ +export function IsUri(value: string): boolean { + return Uri.test(value); +} diff --git a/packages/schema/src/formats/typebox/uri_reference.ts b/packages/schema/src/formats/typebox/uri_reference.ts new file mode 100644 index 000000000..215058ded --- /dev/null +++ b/packages/schema/src/formats/typebox/uri_reference.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const UriReference = + /^(?:[a-z][a-z0-9+\-.]*:(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:\/\/(?:(?:[-a-z0-9._~!$&'()*+,;=:]|%[0-9a-f]{2})*@)?(?:\[(?:(?:(?:[\da-f]{1,4}:){6}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|::(?:[\da-f]{1,4}:){5}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:[\da-f]{1,4})?::(?:[\da-f]{1,4}:){4}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,1}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){3}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,2}[\da-f]{1,4})?::(?:[\da-f]{1,4}:){2}(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,3}[\da-f]{1,4})?::[\da-f]{1,4}:(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,4}[\da-f]{1,4})?::(?:[\da-f]{1,4}:[\da-f]{1,4}|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d))|(?:(?:[\da-f]{1,4}:){0,5}[\da-f]{1,4})?::[\da-f]{1,4}|(?:(?:[\da-f]{1,4}:){0,6}[\da-f]{1,4})?::)|v[0-9a-f]+\.[-a-z0-9._~!$&'()*+,;=:]+)\]|(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)|(?:[-a-z0-9._~!$&'()*+,;=]|%[0-9a-f]{2})*)(?::\d*)?(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*|\/(?:(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?|(?:[-a-z0-9._~!$&'()*+,;=@]|%[0-9a-f]{2})+(?:\/(?:[-a-z0-9._~!$&'()*+,;=:@]|%[0-9a-f]{2})*)*)?)(?:\?(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?(?:#(?:[-a-z0-9._~!$&'()*+,;=:@/?]|%[0-9a-f]{2})*)?$/i; + +/** + * Returns true if the value is a valid Uri Reference. + * @specification https://tools.ietf.org/html/rfc3986 + */ +export function IsUriReference(value: string): boolean { + return UriReference.test(value); +} diff --git a/packages/schema/src/formats/typebox/uri_template.ts b/packages/schema/src/formats/typebox/uri_template.ts new file mode 100644 index 000000000..ca364fe15 --- /dev/null +++ b/packages/schema/src/formats/typebox/uri_template.ts @@ -0,0 +1,40 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +// deno-lint-ignore-file no-control-regex + +const UriTemplate = + /^(?:(?:[^\x00-\x20"<>%\\^`{|}\x7f]|%[0-9a-f]{2})|\{[+#./;?&=,!@|]?(?:[a-z0-9_]|%[0-9a-f]{2})+(?:\.(?:[a-z0-9_]|%[0-9a-f]{2})+)*(?::[1-9]\d{0,3}|\*)?(?:,(?:[a-z0-9_]|%[0-9a-f]{2})+(?:\.(?:[a-z0-9_]|%[0-9a-f]{2})+)*(?::[1-9]\d{0,3}|\*)?)*\})*$/i; + +/** + * Returns true if the value is a valid Uri Template + * @specification https://datatracker.ietf.org/doc/html/rfc6570 + */ +export function IsUriTemplate(value: string): boolean { + return UriTemplate.test(value); +} diff --git a/packages/schema/src/formats/typebox/url.ts b/packages/schema/src/formats/typebox/url.ts new file mode 100644 index 000000000..dfe00c4b8 --- /dev/null +++ b/packages/schema/src/formats/typebox/url.ts @@ -0,0 +1,36 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +/** + * Returns true if the value is a valid Uniform Resource Locator + * @specification https://datatracker.ietf.org/doc/html/rfc3986 + * @specification https://datatracker.ietf.org/doc/html/rfc3987 + */ +export function IsUrl(value: string): boolean { + return URL.canParse(value); +} diff --git a/packages/schema/src/formats/typebox/uuid.ts b/packages/schema/src/formats/typebox/uuid.ts new file mode 100644 index 000000000..cf26ffa6e --- /dev/null +++ b/packages/schema/src/formats/typebox/uuid.ts @@ -0,0 +1,38 @@ +/*-------------------------------------------------------------------------- + +TypeBox + +The MIT License (MIT) + +Copyright (c) 2017-2026 Haydn Paterson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +---------------------------------------------------------------------------*/ + +const Uuid = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i; + +/** + * Returns true if the value is a uuid + * @specification https://www.rfc-editor.org/info/rfc4122/ + * @specification https://www.rfc-editor.org/info/rfc9562/ + */ +export function IsUuid(value: string): boolean { + return Uuid.test(value); +} diff --git a/packages/schema/src/index.ts b/packages/schema/src/index.ts new file mode 100644 index 000000000..c57ede2e9 --- /dev/null +++ b/packages/schema/src/index.ts @@ -0,0 +1,31 @@ +// JSON Schema validation (drafts 2020-12 and 7) without eval: compile() builds a schema into plain closures once. +// The builders return plain JSON Schema objects typed with the values they accept. + +export { + type Annotations, + arr, + bool, + bytes, + int, + type LengthBounds, + lit, + map, + type NumberBounds, + nul, + nullable, + num, + Optional, + obj, + oneOf, + opt, + type Properties, + type Schema, + type Static, + type StaticObject, + str, + tuple, + union, + unknown, +} from "./builders.ts"; +export type { Check, CompileOptions, FormatCheck, JsonSchema, ParseResult, ValidationError } from "./types.ts"; +export { CoercingValidator, compile, Validator } from "./validator.ts"; diff --git a/packages/schema/src/resolver.ts b/packages/schema/src/resolver.ts new file mode 100644 index 000000000..5c8b97543 --- /dev/null +++ b/packages/schema/src/resolver.ts @@ -0,0 +1,203 @@ +// Reference resolution of one compile: schema resources, anchors and JSON Pointers over the root schema and the +// registered documents. A Resolver lives only while a schema compiles (or while its errors are explained). + +import type { CompileOptions, JsonSchema, SchemaObject } from "./types.ts"; + +/** A schema and the base URI its references resolve against. */ +export interface Resolved { + readonly node: unknown; + readonly base: string; +} + +/** The base URI of a root schema without `$id`; relative references resolve against it. */ +export const DEFAULT_BASE = "https://json-schema.invalid/root.json"; + +/** + * Resolves references in the root schema and the registered documents. The schema is indexed on the first + * reference, so a schema without references is never walked for identifiers. + */ +export class Resolver { + readonly draft: "2020-12" | "7"; + readonly rootBase: string; + /** Schema object to the base URI it was indexed under. */ + readonly baseOf = new Map(); + /** URI (with a `#name` fragment) of every `$dynamicAnchor`, to its schema. */ + readonly dynamicAnchors = new Map(); + private readonly root: JsonSchema; + private readonly options: CompileOptions; + private readonly registry = new Map(); + private readonly rejectUnresolved: boolean; + /** The references that named nothing and resolved to `false` (with `unresolvedRefs: "reject"`). */ + private readonly unresolved = new Set(); + private readonly resources = new Map(); + private readonly anchors = new Map(); + private indexed = false; + + constructor(root: JsonSchema, options: CompileOptions) { + this.draft = options.draft === "7" ? "7" : "2020-12"; + this.root = root; + this.options = options; + this.rejectUnresolved = options.unresolvedRefs === "reject"; + this.rootBase = isSchemaObject(root) ? this.baseFor(root, DEFAULT_BASE) : DEFAULT_BASE; + } + + /** + * The base URI of `node`, whose parent has `base`: its `$id` changes it, unless draft 7 makes the `$id` inert. Once + * the schema is indexed, the indexed base (a resolved reference target already has its own). + */ + baseFor(node: SchemaObject, base: string): string { + const indexed = this.baseOf.get(node); + if (indexed !== undefined) return indexed; + const id = node.$id; + if (typeof id !== "string") return base; + if (this.draft === "7" && (typeof node.$ref === "string" || id.startsWith("#"))) return base; + return documentOf(resolveUri(id, base)); + } + + /** Indexes the root schema and the registered documents' URIs (once). */ + ensureIndexed(): void { + if (this.indexed) return; + this.indexed = true; + for (const [uri, schema] of Object.entries(this.options.schemas ?? {})) { + this.registry.set(documentOf(resolveUri(uri, DEFAULT_BASE)), schema); + } + this.resources.set(DEFAULT_BASE, this.root); + this.index(this.root, DEFAULT_BASE); + } + + /** + * The schema a reference names, with the base URI it compiles under. Throws if it names nothing, or, with + * `unresolvedRefs: "reject"`, resolves to `false`, the schema no value passes. + */ + resolve(ref: string, base: string): Resolved { + this.ensureIndexed(); + if (!this.rejectUnresolved) return this.find(ref, base); + try { + return this.find(ref, base); + } catch { + this.unresolved.add(resolveUri(ref, base)); + return { node: false, base }; + } + } + + /** Whether `ref`, from `base`, named nothing and resolved to `false`. */ + isUnresolved(ref: string, base: string): boolean { + return this.unresolved.has(resolveUri(ref, base)); + } + + private find(ref: string, base: string): Resolved { + const uri = resolveUri(ref, base); + const doc = documentOf(uri); + const fragment = uri.length > doc.length ? decodeURIComponent(uri.slice(doc.length + 1)) : ""; + const root = this.resources.has(doc) ? this.resources.get(doc) : this.load(doc); + if (root === undefined) throw new Error(`Unresolved $ref ${ref} (no schema registered for ${doc})`); + if (fragment === "") return { node: root, base: doc }; + if (fragment.startsWith("/")) return this.pointer(root, doc, fragment); + const target = this.anchors.get(`${doc}#${fragment}`); + if (target === undefined) throw new Error(`Unresolved $ref ${ref} (no anchor ${fragment} in ${doc})`); + return { node: target, base: this.baseOf.get(target) ?? doc }; + } + + /** Whether `node` is the root of the schema resource at `base`. */ + isResource(node: object, base: string): boolean { + this.ensureIndexed(); + return this.resources.get(base) === node; + } + + // Walks only keywords that hold subschemas: an `$id` inside `enum`, `const` or an unknown keyword names nothing. + private index(node: unknown, base: string): void { + if (!isSchemaObject(node) || this.baseOf.has(node)) return; + let b = base; + // In draft 7 a `$ref` makes every sibling keyword, `$id` included, inert. + const refOnly = this.draft === "7" && typeof node.$ref === "string"; + if (!refOnly && typeof node.$id === "string") { + if (this.draft === "7" && node.$id.startsWith("#")) this.anchors.set(documentOf(b) + node.$id, node); + else { + const uri = resolveUri(node.$id, b); + b = documentOf(uri); + this.resources.set(b, node); + if (uri.length > b.length + 1) this.anchors.set(uri, node); + } + } + this.baseOf.set(node, b); + if (refOnly) return; + if (this.draft !== "7") { + if (typeof node.$anchor === "string") this.anchors.set(`${b}#${node.$anchor}`, node); + if (typeof node.$dynamicAnchor === "string") { + this.anchors.set(`${b}#${node.$dynamicAnchor}`, node); + this.dynamicAnchors.set(`${b}#${node.$dynamicAnchor}`, node); + } + } + for (const key of SCHEMA_KEYWORDS) this.index(node[key], b); + for (const key of MAP_KEYWORDS) { + const map = node[key]; + if (isSchemaObject(map)) for (const k of Object.keys(map)) this.index(map[k], b); + } + for (const key of ARRAY_KEYWORDS) { + const list = node[key]; + if (Array.isArray(list)) for (const s of list) this.index(s, b); + } + } + + private load(doc: string): JsonSchema | undefined { + const schema = this.registry.get(doc); + if (schema === undefined) return undefined; + this.resources.set(doc, schema); + this.index(schema, doc); + return schema; + } + + private pointer(root: unknown, base: string, fragment: string): Resolved { + let node = root; + let b = base; + for (const raw of fragment.split("/").slice(1)) { + const key = raw.replace(/~1/g, "/").replace(/~0/g, "~"); + node = + typeof node === "object" && node !== null && Object.hasOwn(node, key) + ? (node as SchemaObject)[key] + : undefined; + if (node === undefined) throw new Error(`Unresolved $ref #${fragment} in ${base}`); + if (typeof node === "object" && node !== null) b = this.baseOf.get(node) ?? b; + } + return { node, base: b }; + } +} + +const SCHEMA_KEYWORDS = [ + "additionalProperties", + "propertyNames", + "contains", + "not", + "if", + "then", + "else", + "unevaluatedProperties", + "unevaluatedItems", + "additionalItems", + "contentSchema", + "items", +]; + +const MAP_KEYWORDS = ["properties", "patternProperties", "$defs", "definitions", "dependentSchemas", "dependencies"]; + +const ARRAY_KEYWORDS = ["allOf", "anyOf", "oneOf", "prefixItems", "items"]; + +/** Whether `value` is a schema object (not `true`/`false`, an array or a primitive). */ +export function isSchemaObject(value: unknown): value is SchemaObject { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** `ref` resolved against `base`. URNs and other bases a URL parser rejects take fragment-only references. */ +export function resolveUri(ref: string, base: string): string { + try { + return new URL(ref, base).href; + } catch { + return ref.startsWith("#") ? documentOf(base) + ref : ref; + } +} + +/** `uri` without its fragment. */ +export function documentOf(uri: string): string { + const hash = uri.indexOf("#"); + return hash < 0 ? uri : uri.slice(0, hash); +} diff --git a/packages/schema/src/shared.ts b/packages/schema/src/shared.ts new file mode 100644 index 000000000..42c30152f --- /dev/null +++ b/packages/schema/src/shared.ts @@ -0,0 +1,33 @@ +// Regular expressions shared across all compiled schemas while one still uses them: tool schemas repeat a few +// patterns many times, and a RegExp is large once it has run. The cache holds its values weakly and drops an entry +// once its RegExp is collected, so it keeps nothing alive. + +/** + * The regular expression of a `pattern` source, shared by source. Patterns are ECMAScript regular expressions with + * the `u` flag; a source that is invalid with it (some older schemas escape characters `u` rejects) compiles + * without it. + */ +export function sharedRegex(source: string): RegExp { + const hit = regexes.get(source)?.deref(); + if (hit !== undefined) return hit; + let re: RegExp; + try { + re = new RegExp(source, "u"); + } catch { + re = new RegExp(source); + } + regexes.set(source, new WeakRef(re)); + regexCleanup.register(re, source); + return re; +} + +const regexes = new Map>(); + +const regexCleanup = new FinalizationRegistry((source) => { + if (regexes.get(source)?.deref() === undefined) regexes.delete(source); +}); + +/** The number of cached regular expressions (for tests). */ +export function sharedRegexCount(): number { + return regexes.size; +} diff --git a/packages/schema/src/typebox-static.ts b/packages/schema/src/typebox-static.ts new file mode 100644 index 000000000..ce9f29b51 --- /dev/null +++ b/packages/schema/src/typebox-static.ts @@ -0,0 +1,76 @@ +// The static types of TypeBox-built schemas, read from the type shapes TypeBox 1.x declares (`'~kind'`, `'~optional'`, +// `'~readonly'`, `'~unsafe'`) without importing TypeBox, so `Static` also types tool arguments of TypeBox schemas. +// Only the common kinds are mapped; every other kind (Enum, Tuple, Intersect, Cyclic, Ref, ...) is `unknown`. + +import type { Static } from "./builders.ts"; + +/** The value type of a TypeBox schema type by its `'~kind'`, or its `Type.Unsafe` type; `unknown` for other kinds. */ +export type TypeBoxStatic = S extends { readonly "~kind": infer K } + ? KindStatic + : S extends { readonly "~unsafe": infer T } + ? T + : unknown; + +interface Primitives { + String: string; + Number: number; + Integer: number; + Boolean: boolean; + Null: null; + Unknown: unknown; + Any: any; +} + +type KindStatic = K extends keyof Primitives + ? Primitives[K] + : K extends "Literal" + ? S extends { readonly const: infer V } + ? V + : unknown + : K extends "Array" + ? S extends { readonly items: infer I } + ? Static[] + : unknown + : K extends "Object" + ? S extends { readonly properties: infer P } + ? ObjectStatic

+ : unknown + : K extends "Union" + ? S extends { readonly anyOf: readonly (infer B)[] } + ? Static + : unknown + : K extends "Record" + ? S extends { readonly patternProperties: infer P } + ? Record, Static> + : unknown + : unknown; + +type Optional = { readonly "~optional": true }; + +type Readonly = { readonly "~readonly": true }; + +// As TypeBox: no properties is `object`; `'~optional'` properties may be omitted, `'~readonly'` ones are readonly. +type ObjectStatic

= keyof P extends never + ? object + : Simplify< + { -readonly [K in keyof P as P[K] extends Optional | Readonly ? never : K]: Static } & { + -readonly [K in keyof P as P[K] extends Optional ? (P[K] extends Readonly ? never : K) : never]?: Static< + P[K] + >; + } & { + readonly [K in keyof P as P[K] extends Readonly ? (P[K] extends Optional ? never : K) : never]: Static< + P[K] + >; + } & { + readonly [K in keyof P as P[K] extends Optional ? (P[K] extends Readonly ? K : never) : never]?: Static< + P[K] + >; + } + >; + +type Simplify = { [K in keyof T]: T[K] } & {}; + +// TypeBox's key patterns for integer and number keys; every other pattern types its keys as strings. +type RecordKey = Pattern extends "^-?(?:0|[1-9][0-9]*)$" | "^-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?$" + ? number + : string; diff --git a/packages/schema/src/types.ts b/packages/schema/src/types.ts new file mode 100644 index 000000000..d3f635adb --- /dev/null +++ b/packages/schema/src/types.ts @@ -0,0 +1,49 @@ +// Public types: the schema accepted by compile(), its options, and the errors a failed value produces. + +/** + * A JSON Schema: `true`, `false` or a schema object. Schemas come from tool definitions, MCP servers and other + * untrusted sources, so compile() reads every keyword defensively instead of trusting a static shape. + */ +export type JsonSchema = boolean | object; + +/** A format check: whether `value` (a string) has the format. */ +export type FormatCheck = (value: string) => boolean; + +/** Options of compile(). */ +export interface CompileOptions { + /** The draft whose semantics apply: "2020-12" (default) or "7". */ + readonly draft?: "2020-12" | "7"; + /** + * Format name to check. Without it `format` is an annotation, as draft 2020-12 specifies. The standard formats are + * in `@earendil-works/pi-schema/formats`. + */ + readonly formats?: Readonly>; + /** Schema documents by URI, for `$ref`s to other documents. */ + readonly schemas?: Readonly>; + /** Add `coerce()` and `parse()` to the validator, for loosely typed input (see the README's Coercion section). */ + readonly coerce?: boolean; + /** + * What a `$ref` that names nothing (a missing definition, a document not in `schemas`) does: `"throw"` (default) + * makes compile() throw; `"reject"` compiles it as `false`, so only values at that place fail, with the error + * "unresolved reference". + */ + readonly unresolvedRefs?: "throw" | "reject"; +} + +/** The result of `parse()`: the coerced value when it is valid, else the errors of the coerced value. */ +export type ParseResult = + | { readonly ok: true; readonly value: T } + | { readonly ok: false; readonly errors: ValidationError[] }; + +/** One failure of a value: where it is (a JSON Pointer, "" for the value itself), the failing keyword and a message. */ +export interface ValidationError { + readonly path: string; + readonly keyword: string; + readonly message: string; +} + +/** A schema object as compile() reads it: any keyword, of any value, checked where it is used. */ +export type SchemaObject = { readonly [keyword: string]: unknown }; + +/** A compiled boolean check. */ +export type Check = (value: unknown) => boolean; diff --git a/packages/schema/src/validator.ts b/packages/schema/src/validator.ts new file mode 100644 index 000000000..65781e5dd --- /dev/null +++ b/packages/schema/src/validator.ts @@ -0,0 +1,93 @@ +// compile(): a schema to a Validator, whose `check` is the fast boolean path and whose `errors` explains failures. + +import type { Static } from "./builders.ts"; +import { Coercer } from "./coerce.ts"; +import { Compiler, RESTART } from "./compiler.ts"; +import { Explainer } from "./explain.ts"; +import type { Check, CompileOptions, JsonSchema, ParseResult, ValidationError } from "./types.ts"; + +/** + * Compiles `schema` into a Validator. Throws if the schema is malformed or a `$ref` names nothing (register other + * documents with `options.schemas`). The schema is read, never modified. With `coerce: true` the validator also + * coerces and parses loosely typed input. + */ +export function compile( + schema: S, + options: CompileOptions & { readonly coerce: true }, +): CoercingValidator>; +export function compile(schema: S, options?: CompileOptions): Validator>; +export function compile( + schema: S, + options: CompileOptions = NO_OPTIONS, +): Validator> | CoercingValidator> { + const check = compileCheck(schema, options); + return options.coerce === true + ? new CoercingValidator(schema, options, check) + : new Validator(schema, options, check); +} + +const NO_OPTIONS: CompileOptions = Object.freeze({}); + +/** A compiled schema whose valid values have type `T` (`unknown` unless the schema was built with the builders). */ +export class Validator { + /** Whether `value` is valid. A plain function: it may be passed around without its validator. */ + readonly check: (value: unknown) => value is T; + protected readonly schema: JsonSchema; + protected readonly options: CompileOptions; + private explainer: Explainer | undefined = undefined; + + constructor(schema: JsonSchema, options: CompileOptions, check: Check) { + // A compiled check is exactly the test that a value has the schema's static type. + this.check = check as (value: unknown) => value is T; + this.schema = schema; + this.options = options; + } + + /** + * Why `value` is invalid: one error per failing keyword, `[]` when it is valid. Built on the first failure and + * kept, so a validator whose values always pass never pays for it. + */ + errors(value: unknown): ValidationError[] { + if (this.check(value)) return []; + this.explainer ??= new Explainer(compiler(this.schema, this.options), this.schema); + return this.explainer.errors(value); + } +} + +/** A validator compiled with `coerce: true`: it also coerces loosely typed input towards the schema. */ +export class CoercingValidator extends Validator { + private coercer: Coercer | undefined = undefined; + + /** + * `value` coerced towards the schema: numeric strings to numbers, `"true"` to `true`, a single value to a one-item + * array, an optional `null` dropped, the first union branch the value converts to (the README lists every rule). + * Only the changed paths are copied; `value` comes back itself when nothing changes, and is never modified. Values + * that cannot be coerced stay as they are. The rules are built on the first call. + */ + coerce(value: unknown): unknown { + this.coercer ??= new Coercer(compiler(this.schema, this.options), this.schema); + return this.coercer.coerce(value); + } + + /** Coerces `value`, then checks it: the coerced value when it is valid, else its errors. */ + parse(value: unknown): ParseResult { + const coerced = this.coerce(value); + return this.check(coerced) ? { ok: true, value: coerced } : { ok: false, errors: this.errors(coerced) }; + } +} + +function compileCheck(schema: JsonSchema, options: CompileOptions): Check { + return compiler(schema, options).root(); +} + +// Most schemas compile without tracking the dynamic scope; one with a bookended $dynamicRef compiles again with it. +function compiler(schema: JsonSchema, options: CompileOptions): Compiler { + const plain = new Compiler(schema, options, false); + try { + plain.root(); + return plain; + } catch (error) { + if (error !== RESTART) throw error; + return new Compiler(schema, options, true); + } +} diff --git a/packages/schema/test/bench/all.ts b/packages/schema/test/bench/all.ts new file mode 100644 index 000000000..96ce485c4 --- /dev/null +++ b/packages/schema/test/bench/all.ts @@ -0,0 +1,40 @@ +// Runs every benchmark, each in its own process, and writes the results as JSON next to the printed tables. +// Speed runs twice: normally and with code generation from strings disabled, as under a strict CSP. +// The children run with ATA_NO_NATIVE=1 (see candidates.ts). +// Run: npm run bench [-- --out ] (default: pi-schema-bench-results.json in the temp directory) + +import { execFileSync } from "node:child_process"; +import { readFileSync, rmSync, writeFileSync } from "node:fs"; +import { cpus, tmpdir } from "node:os"; +import { join } from "node:path"; + +const dir = import.meta.dirname; +const outIndex = process.argv.indexOf("--out"); +const out = outIndex >= 0 ? process.argv[outIndex + 1] : join(tmpdir(), "pi-schema-bench-results.json"); + +const runs: [string, string, string[]][] = [ + ["conformance", "conformance.ts", []], + ["speed", "speed.ts", []], + ["speedNoEval", "speed.ts", ["--disallow-code-generation-from-strings"]], + ["setup", "setup.ts", []], + ["memory", "memory.ts", []], + ["size", "size.ts", []], +]; + +const results: Record = { + node: process.version, + cpu: cpus()[0]?.model ?? "unknown", + date: new Date().toISOString(), +}; +for (const [key, file, flags] of runs) { + const json = join(tmpdir(), `pi-schema-bench-${key}.json`); + console.log(`\n## ${key}${flags.length > 0 ? ` (${flags.join(" ")})` : ""}\n`); + execFileSync(process.execPath, [...flags, join(dir, file), "--json", json], { + stdio: "inherit", + env: { ...process.env, ATA_NO_NATIVE: "1" }, + }); + results[key] = JSON.parse(readFileSync(json, "utf8")); + rmSync(json); +} +writeFileSync(out, JSON.stringify(results, null, "\t")); +console.log(`\nResults: ${out}`); diff --git a/packages/schema/test/bench/candidates.ts b/packages/schema/test/bench/candidates.ts new file mode 100644 index 000000000..ce0c6d4cf --- /dev/null +++ b/packages/schema/test/bench/candidates.ts @@ -0,0 +1,205 @@ +// The validators the benchmarks compare, each behind the same interface: compile a schema, then check values. +// Every library is used through its fastest boolean path and keeps the object it returns, as an application would. +// Run the benchmarks through `npm run bench`, which sets ATA_NO_NATIVE=1: ata-validator's optional native addon only +// serves its buffer APIs, and the verdicts measured here are JavaScript either way. + +import { Validator as CfValidator } from "@cfworker/json-schema"; +import { fromJsonSchema } from "@oh-my-pi/omptype"; +import Ajv7Module from "ajv"; +import Ajv2020Module from "ajv/dist/2020.js"; +import draft07MetaSchema from "ajv/dist/refs/json-schema-draft-07.json" with { type: "json" }; +import addFormatsModule from "ajv-formats"; +import { Validator as AtaValidator } from "ata-validator"; +import { Compile } from "typebox/compile"; +import { ResetExternal, ResetFunctions } from "typebox/schema"; +import { Settings } from "typebox/system"; +import { compile, type FormatCheck, type JsonSchema } from "../../src/index.ts"; +import { nativeCases, nativeZod, zodFromJsonSchema } from "./zod.ts"; + +/** How a benchmark compiles a schema. */ +export interface PrepareOptions { + readonly draft: "2020-12" | "7"; + readonly schemas?: Readonly>; + readonly formats?: Readonly>; + /** The schema is a JSON-Schema-Test-Suite schema (for libraries that take native schemas otherwise). */ + readonly suite?: boolean; +} + +/** A compiled schema: its boolean check and the object the library returned (kept alive by the caller). */ +export interface Prepared { + readonly check: (value: unknown) => boolean; + readonly validator: unknown; +} + +/** A library under comparison. */ +export interface Candidate { + readonly name: string; + readonly prepare: (schema: JsonSchema, options: PrepareOptions) => Prepared; +} + +/** Whether this process runs with code generation from strings disabled, as under a strict CSP. */ +export const noEval = process.execArgv.includes("--disallow-code-generation-from-strings"); + +const piSchema: Candidate = { + name: "pi-schema", + prepare(schema, options) { + const validator = compile(schema, options); + return { check: validator.check, validator }; + }, +}; + +// TypeBox keeps its last build in module-level accumulators; reset them so no build state is retained between schemas. +function typeboxCompile(schema: JsonSchema, options: PrepareOptions, accelerate: boolean) { + Settings.Set({ useAcceleration: accelerate }); + try { + return Compile((options.schemas ?? {}) as never, schema as never); + } finally { + ResetFunctions(); + ResetExternal(); + Settings.Set({ useAcceleration: true }); + } +} + +const typebox: Candidate = { + name: noEval ? "typebox (no eval: interpreter)" : "typebox", + prepare(schema, options) { + const validator = typeboxCompile(schema, options, true); + return { check: (v) => validator.Check(v), validator }; + }, +}; + +const typeboxInterpreter: Candidate = { + name: "typebox interpreter", + prepare(schema, options) { + const validator = typeboxCompile(schema, options, false); + return { check: (v) => validator.Check(v), validator }; + }, +}; + +const cfworker: Candidate = { + name: "cfworker", + prepare(schema, options) { + const validator = new CfValidator(schema as never, options.draft, true); + for (const [uri, remote] of Object.entries(options.schemas ?? {})) validator.addSchema(remote as never, uri); + return { check: (v) => validator.validate(v).valid, validator }; + }, +}; + +// omptype compiles with new Function on the first allows() call. Without eval that call throws EvalError once per +// schema; omptype then keeps validating through its interpreter, so the benchmark absorbs that one throw. +const omptype: Candidate = { + name: noEval ? "omptype (no eval: interpreter)" : "omptype", + prepare(schema) { + const validator = fromJsonSchema(schema); + if (noEval) { + try { + validator.allows(null); + } catch (error) { + if (!(error instanceof EvalError)) throw error; + } + } + return { check: (v) => validator.allows(v), validator }; + }, +}; + +// Ajv's CommonJS modules export the class as module.exports and as `default`; the latter is what its types describe. +const Ajv2020 = Ajv2020Module.default; +const Ajv7 = Ajv7Module.default; +const addFormats = addFormatsModule.default; +type AjvInstance = InstanceType; + +// One Ajv per draft, as an application keeps it. Ajv2020 also gets the draft-07 meta-schema, so tool schemas that +// declare draft-07 compile. strict: false so schemas written for other tools compile; logger: false keeps Ajv from +// printing generated code when compiling fails without eval. +function newAjv(draft: "2020-12" | "7", formats: boolean): AjvInstance { + const options = { strict: false, validateFormats: formats, logger: false as const }; + const ajv = draft === "7" ? new Ajv7(options) : new Ajv2020(options); + if (draft !== "7") ajv.addMetaSchema(draft07MetaSchema); + if (formats) addFormats(ajv); + return ajv; +} + +const sharedAjv = new Map(); + +// Ajv compiles with new Function and has no interpreter: without eval, compile() throws EvalError. +const ajv: Candidate = { + name: noEval ? "ajv (no eval: fails)" : "ajv", + prepare(schema, options) { + let instance: AjvInstance; + if (options.schemas === undefined) { + const key = `${options.draft} ${options.formats !== undefined}`; + instance = sharedAjv.get(key) ?? newAjv(options.draft, options.formats !== undefined); + sharedAjv.set(key, instance); + } else { + // The suite reuses $ids across groups, which one shared instance would reject: a fresh one per schema. + instance = newAjv(options.draft, options.formats !== undefined); + for (const [uri, remote] of Object.entries(options.schemas)) { + // Ajv ships its own meta-schemas; adding them again under the same id throws. + if (instance.getSchema(uri) === undefined) { + try { + instance.addSchema(remote as object, uri); + } catch { + // A remote whose $id Ajv already registered under another key. + } + } + } + } + const validator = instance.compile(schema as object); + return { check: (v) => validator(v) as boolean, validator }; + }, +}; + +// ata-validator: its defaults fill `default` values into the input and assert `format`; both are turned off to match +// the other candidates (format checks only in the format run, with its own checks). It reads the draft only from +// $schema, so a draft 7 schema without one gets it declared. +function ataValidator(schema: JsonSchema, options: PrepareOptions, engine: "interpreter" | undefined) { + const declared = + options.draft === "7" && typeof schema === "object" && !("$schema" in schema) + ? { $schema: "http://json-schema.org/draft-07/schema#", ...schema } + : schema; + return new AtaValidator(declared, { + useDefaults: false, + assertFormat: options.formats !== undefined, + schemas: (options.schemas ?? {}) as Record, + logger: false, + ...(engine === undefined ? {} : { engine }), + }); +} + +// Generated code, or its interpreted engine where code generation is blocked. +const ata: Candidate = { + name: noEval ? "ata (no eval: interpreter)" : "ata", + prepare(schema, options) { + const validator = ataValidator(schema, options, undefined); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Its eval-free engine, a tree of closures, chosen explicitly: what it runs on for schemas from outside. +const ataInterpreter: Candidate = { + name: "ata interpreter", + prepare(schema, options) { + const validator = ataValidator(schema, options, "interpreter"); + return { check: (v) => validator.isValidObject(v), validator }; + }, +}; + +// Zod: native Zod schemas for the speed cases and the corpus, z.fromJSONSchema only for the suite. Its object parser +// generates code with new Function when it can and parses without it otherwise; safeParse is its boolean path. +const zod: Candidate = { + name: noEval ? "zod (no eval: no JIT)" : "zod", + prepare(schema, options) { + const validator = options.suite + ? zodFromJsonSchema(schema, options.draft) + : (nativeCases.get(schema) ?? nativeZod(schema as Record)); + return { check: (v) => validator.safeParse(v).success, validator }; + }, +}; + +/** Every candidate, ours first. */ +export const candidates: readonly Candidate[] = [piSchema, typebox, ajv, ata, ataInterpreter, zod, cfworker, omptype]; + +/** Candidates by name, including TypeBox's interpreter (what TypeBox runs when eval is unavailable). */ +export const byName: ReadonlyMap = new Map( + [...candidates, typeboxInterpreter].map((c) => [c.name.split(" (")[0], c]), +); diff --git a/packages/schema/test/bench/cases.ts b/packages/schema/test/bench/cases.ts new file mode 100644 index 000000000..d6821df1e --- /dev/null +++ b/packages/schema/test/bench/cases.ts @@ -0,0 +1,261 @@ +// Speed cases: realistic schemas with a valid and an invalid value each. Argument checks shaped like Chord's +// contracts (closed objects, tagged unions), pi's built-in tool schemas, and MCP tool schemas with $ref and anyOf. + +import type { JsonSchema } from "../../src/index.ts"; + +/** A schema, the draft it is written in, and one valid and one invalid value. */ +export interface Case { + readonly name: string; + readonly schema: JsonSchema; + readonly valid: unknown; + readonly invalid: unknown; +} + +type Props = Record; +const closed = (properties: Props, optional: string[] = []) => ({ + type: "object", + properties, + required: Object.keys(properties).filter((k) => !optional.includes(k)), + additionalProperties: false, +}); +const str = (o: object = {}) => ({ type: "string", ...o }); +const int = (o: object = {}) => ({ type: "integer", ...o }); +const num = (o: object = {}) => ({ type: "number", ...o }); +const lit = (v: string) => ({ const: v }); +const arr = (items: object, o: object = {}) => ({ type: "array", items, ...o }); +const uuid = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"; + +const block = { + anyOf: [ + closed({ kind: lit("text"), text: str() }), + closed({ kind: lit("image"), src: str(), width: int({ minimum: 0 }), height: int({ minimum: 0 }), alt: str() }, [ + "alt", + ]), + closed({ kind: lit("code"), lang: { anyOf: [lit("ts"), lit("js"), lit("py")] }, source: str() }), + ], +}; + +const document = closed( + { + id: str({ pattern: uuid }), + title: str({ minLength: 1, maxLength: 200 }), + tags: arr(str(), { maxItems: 16 }), + author: closed({ name: str(), email: str() }, ["email"]), + blocks: arr(block), + score: num({ minimum: 0, maximum: 1 }), + draft: { type: "boolean" }, + }, + ["score", "draft"], +); + +const documentValue = { + id: "01890a5d-ac96-774b-bcce-b302099a8057", + title: "Quarterly notes", + tags: ["a", "b", "c"], + author: { name: "Mario", email: "m@example.com" }, + blocks: [ + { kind: "text", text: "Hello" }, + { kind: "image", src: "a.png", width: 640, height: 480 }, + { kind: "code", lang: "ts", source: "const x = 1;" }, + { kind: "text", text: "World" }, + { kind: "image", src: "b.png", width: 10, height: 20, alt: "b" }, + ], + score: 0.5, +}; + +const id = int({ minimum: 0 }); +const message = { + anyOf: [ + closed({ + kind: lit("hello"), + protocol: int({ minimum: 0 }), + incarnation: str({ pattern: uuid }), + calls: int({ minimum: 1 }), + }), + closed( + { + kind: lit("hold"), + hold: id, + type: str(), + version: int({ minimum: 1 }), + fingerprint: str({ pattern: "^[0-9a-f]{16}$" }), + params: {}, + address: closed({ kind: str(), params: {} }), + chain: arr(str({ pattern: uuid }), { maxItems: 64 }), + }, + ["chain"], + ), + closed({ kind: lit("release"), hold: id }), + closed({ kind: lit("call"), call: id, hold: id, method: id, args: {} }, ["args"]), + closed({ kind: lit("abort"), call: id }), + ], +}; + +const tool = (properties: Props, required: string[]) => ({ type: "object", properties, required }); + +const mcpIssue = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + owner: str({ minLength: 1, pattern: "^[A-Za-z0-9-]+$" }), + repo: str({ minLength: 1, pattern: "^[A-Za-z0-9._-]+$" }), + title: str({ minLength: 1 }), + body: str(), + labels: arr(str()), + assignees: arr(str()), + milestone: { anyOf: [{ type: "number" }, { type: "null" }] }, + state: { type: "string", enum: ["open", "closed"] }, + }, + required: ["owner", "repo", "title"], + additionalProperties: false, +}; + +const mcpEdit = { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + $defs: { + Range: closed({ start: int({ minimum: 1 }), end: int({ minimum: 1 }) }), + Replace: closed({ oldText: str(), newText: str() }), + LineEdit: closed({ range: { $ref: "#/$defs/Range" }, text: str() }), + }, + properties: { + path: str({ pattern: "^(/|\\./|[A-Za-z]:\\\\)?[^\\u0000]+$" }), + edits: arr({ anyOf: [{ $ref: "#/$defs/Replace" }, { $ref: "#/$defs/LineEdit" }] }, { minItems: 1 }), + dryRun: { type: "boolean" }, + }, + required: ["path", "edits"], + additionalProperties: false, +}; + +const mcpPage = { + type: "object", + $defs: { + RichText: closed( + { + type: { enum: ["text", "mention"] }, + content: str({ maxLength: 2000 }), + annotations: closed({ bold: { type: "boolean" }, italic: { type: "boolean" }, color: str() }, [ + "bold", + "italic", + "color", + ]), + }, + ["annotations"], + ), + Block: { + type: "object", + properties: { + type: { enum: ["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"] }, + rich_text: arr({ $ref: "#/$defs/RichText" }), + checked: { type: "boolean" }, + children: arr({ $ref: "#/$defs/Block" }), + }, + required: ["type", "rich_text"], + additionalProperties: false, + }, + }, + properties: { + parent: { oneOf: [closed({ page_id: str({ pattern: uuid }) }), closed({ database_id: str({ pattern: uuid }) })] }, + title: str({ minLength: 1 }), + due: str({ pattern: "^\\d{4}-\\d{2}-\\d{2}$" }), + children: arr({ $ref: "#/$defs/Block" }, { maxItems: 100 }), + }, + required: ["parent", "title"], + additionalProperties: false, +}; + +const rt = (content: string) => ({ + type: "text", + content, + annotations: { bold: false, italic: false, color: "default" }, +}); +const pageValue = { + parent: { page_id: "01890a5d-ac96-774b-bcce-b302099a8057" }, + title: "Plan", + due: "2026-11-01", + children: [ + { type: "heading_1", rich_text: [rt("Plan")] }, + { + type: "bulleted_list_item", + rich_text: [rt("one")], + children: [{ type: "to_do", rich_text: [rt("sub")], checked: true }], + }, + { type: "paragraph", rich_text: [rt("a"), rt("b"), rt("c")] }, + ], +}; +const pageInvalid = structuredClone(pageValue); +pageInvalid.children[1].children = [{ type: "to_do", rich_text: [rt("sub")], checked: "yes" as unknown as boolean }]; + +/** The speed cases. */ +export const cases: readonly Case[] = [ + { name: "small closed object", schema: closed({ text: str() }), valid: { text: "hello" }, invalid: { text: 1 } }, + { + name: "nested object with tagged unions", + schema: document, + valid: documentValue, + invalid: { + ...documentValue, + blocks: [...documentValue.blocks.slice(0, 4), { kind: "image", src: "b.png", width: -1, height: 20 }], + }, + }, + { + name: "message union of 5", + schema: message, + valid: { kind: "call", call: 7, hold: 3, method: 1, args: { a: 1 } }, + invalid: { kind: "call", call: 7, hold: 3, method: 1, extra: true }, + }, + { + name: "tool: read", + schema: tool({ path: str({ description: "Path to the file" }), offset: num(), limit: num() }, ["path"]), + valid: { path: "src/a.ts", offset: 10, limit: 50 }, + invalid: { offset: 1 }, + }, + { + name: "tool: edit", + schema: tool({ path: str(), edits: arr(tool({ oldText: str(), newText: str() }, ["oldText", "newText"])) }, [ + "path", + "edits", + ]), + valid: { + path: "a.ts", + edits: [ + { oldText: "a", newText: "b" }, + { oldText: "c", newText: "d" }, + ], + }, + invalid: { path: "a.ts", edits: [{ oldText: "a" }] }, + }, + { + name: "tool: bash", + schema: tool({ command: str(), timeout: num() }, ["command"]), + valid: { command: "ls -la", timeout: 30 }, + invalid: { command: 5 }, + }, + { + name: "MCP create_issue (pattern, anyOf, enum)", + schema: mcpIssue, + valid: { + owner: "earendil-works", + repo: "pi", + title: "Bug", + body: "...", + labels: ["bug"], + milestone: null, + state: "open", + }, + invalid: { owner: "earendil works", repo: "pi", title: "Bug" }, + }, + { + name: "MCP edit_file ($ref, anyOf)", + schema: mcpEdit, + valid: { + path: "./src/a.ts", + edits: [ + { oldText: "a", newText: "b" }, + { range: { start: 1, end: 3 }, text: "x" }, + ], + }, + invalid: { path: "./src/a.ts", edits: [{ range: { start: 0, end: 3 }, text: "x" }] }, + }, + { name: "MCP create_page (recursive $ref, oneOf)", schema: mcpPage, valid: pageValue, invalid: pageInvalid }, +]; diff --git a/packages/schema/test/bench/conformance.ts b/packages/schema/test/bench/conformance.ts new file mode 100644 index 000000000..9d8977d76 --- /dev/null +++ b/packages/schema/test/bench/conformance.ts @@ -0,0 +1,56 @@ +// Conformance of every candidate on the vendored JSON-Schema-Test-Suite (drafts 2020-12 and 7), by section: +// core, optional and optional/format. Every candidate gets the remote documents and meta-schemas; format tests run +// with pi-schema's format checks (the other libraries check formats on their own or not at all). Zod goes through +// z.fromJSONSchema, which refuses some keywords and every remote $ref: a schema it refuses fails all its tests. +// Run: node test/bench/conformance.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { formats } from "../../src/formats/index.ts"; +import { suiteFiles, suiteSchemas } from "../suite.ts"; +import { candidates } from "./candidates.ts"; + +/** Passed and total tests by section ("draft2020-12 core", ...) and candidate name. */ +export type ConformanceResults = Record }>; + +const results: ConformanceResults = {}; +for (const file of suiteFiles()) { + const kind = file.name.includes("/optional/format/") + ? "format" + : file.name.includes("/optional/") + ? "optional" + : "core"; + const section = `${file.name.split("/")[0]} ${kind}`; + results[section] ??= { total: 0, passed: Object.fromEntries(candidates.map((c) => [c.name, 0])) }; + for (const group of file.groups) { + results[section].total += group.tests.length; + for (const candidate of candidates) { + let check: ((v: unknown) => boolean) | undefined; + try { + check = candidate.prepare(structuredClone(group.schema), { + draft: file.draft, + schemas: suiteSchemas(), + formats: kind === "format" ? formats : undefined, + suite: true, + }).check; + } catch { + continue; // A schema the library cannot compile fails all its tests. + } + for (const test of group.tests) { + try { + if (check(test.data) === test.valid) results[section].passed[candidate.name]++; + } catch { + // A throw while checking fails the test. + } + } + } + } +} + +const names = candidates.map((c) => c.name); +console.log(`| section | tests | ${names.join(" | ")} |`); +console.log(`|---|---:|${names.map(() => "---:").join("|")}|`); +for (const [section, r] of Object.entries(results)) { + console.log(`| ${section} | ${r.total} | ${names.map((n) => r.passed[n]).join(" | ")} |`); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/corpus.ts b/packages/schema/test/bench/corpus.ts new file mode 100644 index 000000000..2a985db8e --- /dev/null +++ b/packages/schema/test/bench/corpus.ts @@ -0,0 +1,122 @@ +// A deterministic corpus of distinct tool-like JSON Schemas with one valid value each: random property names, +// descriptions, leaf constraints, enums, nested objects, arrays and some $defs/$ref. No two schemas are copies. + +/** A schema and a value it accepts. */ +export interface CorpusItem { + readonly schema: Record; + readonly value: unknown; +} + +const SYLLABLES = [ + "ka", + "lo", + "mi", + "re", + "su", + "ta", + "ne", + "vo", + "pi", + "da", + "ru", + "xe", + "bo", + "fi", + "gu", + "ha", + "jo", + "ze", +]; +const PATTERNS: readonly (readonly [string, string])[] = [ + ["^[a-z0-9-]+$", "a-1"], + ["^[A-Za-z0-9._-]+$", "A.b_c"], + ["^\\d{4}-\\d{2}-\\d{2}$", "2026-10-31"], + ["^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", "01890a5d-ac96-774b-bcce-b302099a8057"], + ["^#[0-9a-f]{6}$", "#00ff00"], +]; + +/** `count` distinct schemas, the same for the same seed. */ +export function corpus(count: number, seed = 42): CorpusItem[] { + const random = mulberry32(seed); + const int = (lo: number, hi: number) => lo + Math.floor(random() * (hi - lo + 1)); + const pick = (list: readonly T[]): T => list[Math.floor(random() * list.length)]; + const word = () => Array.from({ length: int(2, 4) }, () => pick(SYLLABLES)).join(""); + const sentence = () => `${Array.from({ length: int(4, 12) }, word).join(" ")}.`; + + function leaf(depth: number): [Record, unknown] { + const r = random(); + if (r < 0.3) return [{ type: "string", description: sentence() }, word()]; + if (r < 0.4) { + const min = int(0, 3); + return [ + { type: "string", minLength: min, maxLength: min + int(5, 200), description: sentence() }, + "x".repeat(min + 1), + ]; + } + if (r < 0.47) { + const [pattern, value] = pick(PATTERNS); + return [{ type: "string", pattern, description: sentence() }, value]; + } + if (r < 0.55) { + const values = Array.from({ length: int(2, 5) }, word); + return [{ type: "string", enum: values, description: sentence() }, values[0]]; + } + if (r < 0.65) { + const min = int(0, 10); + return [{ type: "integer", minimum: min, maximum: min + int(10, 10000), description: sentence() }, min + 1]; + } + if (r < 0.72) return [{ type: "number", description: sentence() }, random() * 100]; + if (r < 0.8) return [{ type: "boolean", description: sentence() }, random() < 0.5]; + if (r < 0.88) { + return [{ type: "array", items: { type: "string" }, maxItems: int(1, 50), description: sentence() }, [word()]]; + } + if (depth < 2) return object(depth + 1, int(2, 5)); + return [{ type: "string", description: sentence() }, word()]; + } + + function object(depth: number, n: number): [Record, Record] { + const properties: Record> = {}; + const value: Record = {}; + const required: string[] = []; + for (let i = 0; i < n; i++) { + const name = word() + (i > 0 ? i : ""); + const [schema, v] = leaf(depth); + properties[name] = schema; + if (random() < 0.6) { + required.push(name); + value[name] = v; + } else if (random() < 0.5) value[name] = v; + } + const schema: Record = { type: "object", properties, required }; + if (random() < 0.5) schema.additionalProperties = false; + return [schema, value]; + } + + const items: CorpusItem[] = []; + for (let i = 0; i < count; i++) { + const [schema, value] = object(0, int(2, 10)); + const properties = schema.properties as Record>; + if (random() < 0.2) { + // Move one nested object into $defs and reference it, as MCP servers built with zod often do. + const key = Object.keys(properties).find((k) => properties[k].type === "object"); + if (key !== undefined) { + const name = `${word()}Def`; + schema.$defs = { [name]: properties[key] }; + properties[key] = { $ref: `#/$defs/${name}` }; + } + } + schema.description = sentence(); + items.push({ schema, value }); + } + return items; +} + +function mulberry32(seed: number): () => number { + let state = seed; + return () => { + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/packages/schema/test/bench/memory.ts b/packages/schema/test/bench/memory.ts new file mode 100644 index 000000000..9f2c1546b --- /dev/null +++ b/packages/schema/test/bench/memory.ts @@ -0,0 +1,73 @@ +// Retained heap of 3,200 distinct compiled schemas per candidate. Each candidate runs in its own process with +// --expose-gc: the schemas and values are built before the baseline, every library's returned validator is kept, +// and the heap is read after forced collections, once after compiling and once after checking each value 3 times +// (when libraries that compile lazily, like omptype, have compiled). +// Run: node test/bench/memory.ts [--json ] + +import { execFileSync } from "node:child_process"; +import { writeFileSync } from "node:fs"; +import { byName, candidates } from "./candidates.ts"; +import { corpus } from "./corpus.ts"; + +const COUNT = Number(process.env.COUNT ?? 3200); + +/** MB retained by the schemas themselves, and by each candidate after compiling and after checking. */ +export interface MemoryResults { + readonly count: number; + readonly schemasMB: number; + readonly candidates: Record; +} + +const child = process.argv.indexOf("--child"); +if (child >= 0) measure(process.argv[child + 1]); +else run(); + +function run(): void { + const results: MemoryResults = { count: COUNT, schemasMB: 0, candidates: {} }; + const names = [...candidates.map((c) => c.name), "typebox interpreter"]; + for (const name of names) { + const output = execFileSync(process.execPath, ["--expose-gc", import.meta.filename, "--child", name], { + encoding: "utf8", + env: { ...process.env, COUNT: String(COUNT) }, + }); + const result = JSON.parse(output); + Object.assign(results, { schemasMB: result.schemasMB }); + results.candidates[name] = { compiledMB: result.compiledMB, checkedMB: result.checkedMB }; + } + console.log(`${COUNT} distinct schemas take ${results.schemasMB.toFixed(1)} MB themselves (not counted below).\n`); + console.log("| candidate | after compiling | after checking |"); + console.log("|---|---:|---:|"); + for (const [name, r] of Object.entries(results.candidates)) { + console.log(`| ${name} | ${r.compiledMB.toFixed(1)} MB | ${r.checkedMB.toFixed(1)} MB |`); + } + const out = process.argv.indexOf("--json"); + if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); +} + +function measure(name: string): void { + const candidate = byName.get(name); + if (candidate === undefined) throw new Error(`Unknown candidate ${name}`); + const gc = (globalThis as { gc?: () => void }).gc; + if (gc === undefined) throw new Error("Run with --expose-gc"); + const heap = () => { + for (let i = 0; i < 4; i++) gc(); + return process.memoryUsage().heapUsed; + }; + // Warm the library's own code on another corpus, so its lazily compiled functions are not counted. + for (const { schema, value } of corpus(100, 7)) { + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + for (let i = 0; i < 4; i++) check(value); + } + const empty = heap(); + const items = corpus(COUNT, 42); + const base = heap(); + const prepared = items.map(({ schema }) => candidate.prepare(schema, { draft: "2020-12" })); + const compiled = heap(); + for (let i = 0; i < items.length; i++) for (let k = 0; k < 3; k++) prepared[i].check(items[i].value); + const checked = heap(); + (globalThis as { keep?: unknown }).keep = [items, prepared]; + const mb = (bytes: number) => bytes / 1048576; + console.log( + JSON.stringify({ schemasMB: mb(base - empty), compiledMB: mb(compiled - base), checkedMB: mb(checked - base) }), + ); +} diff --git a/packages/schema/test/bench/setup.ts b/packages/schema/test/bench/setup.ts new file mode 100644 index 000000000..ccf188c2b --- /dev/null +++ b/packages/schema/test/bench/setup.ts @@ -0,0 +1,51 @@ +// Setup cost: compiling a schema the library has not seen, and the time of its first checks. Uses distinct +// schemas from the corpus after warming each library's own code, so it measures a new schema, not a new process. +// Run: node test/bench/setup.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { candidates } from "./candidates.ts"; +import { corpus } from "./corpus.ts"; + +const COUNT = Number(process.env.COUNT ?? 1000); + +/** Mean µs per schema: compile, then each of the first three checks and the mean of checks 4-10. */ +export type SetupResults = Record; + +const results: SetupResults = {}; +for (const candidate of candidates) { + for (const { schema, value } of corpus(200, 7)) { + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + for (let i = 0; i < 10; i++) check(value); + } + const items = corpus(COUNT, 99); + let compile = 0; + const checks = [0, 0, 0, 0]; + for (const { schema, value } of items) { + let start = process.hrtime.bigint(); + const { check } = candidate.prepare(schema, { draft: "2020-12" }); + compile += Number(process.hrtime.bigint() - start); + for (let k = 0; k < 3; k++) { + start = process.hrtime.bigint(); + check(value); + checks[k] += Number(process.hrtime.bigint() - start); + } + start = process.hrtime.bigint(); + for (let k = 3; k < 10; k++) check(value); + checks[3] += Number(process.hrtime.bigint() - start) / 7; + } + const us = (ns: number) => ns / COUNT / 1000; + results[candidate.name] = { + compileUs: us(compile), + checkUs: [us(checks[0]), us(checks[1]), us(checks[2]), us(checks[3])], + }; +} + +console.log("| candidate | compile | check 1 | check 2 | check 3 | checks 4-10 |"); +console.log("|---|---:|---:|---:|---:|---:|"); +for (const [name, r] of Object.entries(results)) { + console.log( + `| ${name} | ${r.compileUs.toFixed(1)} µs | ${r.checkUs.map((x) => `${x.toFixed(2)} µs`).join(" | ")} |`, + ); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/size.ts b/packages/schema/test/bench/size.ts new file mode 100644 index 000000000..46a09e3d3 --- /dev/null +++ b/packages/schema/test/bench/size.ts @@ -0,0 +1,50 @@ +// Bundle size of what a validator needs from each library, as a browser app ships it: esbuild bundles and minifies the +// import for the browser platform (one file, no code splitting), then gzip -9. +// Run: node test/bench/size.ts [--json ] + +import { writeFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; +import { build } from "esbuild"; + +const src = new URL("../../src/", import.meta.url).pathname; + +const entries: Record = { + "pi-schema": `export { compile } from ${JSON.stringify(`${src}index.ts`)};`, + "pi-schema + formats": `export { compile } from ${JSON.stringify(`${src}index.ts`)}; export { formats } from ${JSON.stringify(`${src}formats/index.ts`)};`, + typebox: `export { Compile } from "typebox/compile";`, + cfworker: `export { Validator } from "@cfworker/json-schema";`, + ajv: `export { default } from "ajv/dist/2020.js";`, + "ajv + ajv-formats": `export { default } from "ajv/dist/2020.js"; export { default as addFormats } from "ajv-formats";`, + "ata-validator": `export { Validator } from "ata-validator";`, + // Zod's documented `import * as z` (a named `{ z }` import keeps every locale, about 90 KB gzipped). Its core with + // native schemas, and with z.fromJSONSchema for JSON Schema. + zod: `import * as z from "zod"; export const check = (v: unknown) => z.object({ a: z.string() }).safeParse(v).success;`, + "zod + fromJSONSchema": `import * as z from "zod"; export const fromJSONSchema = z.fromJSONSchema;`, + omptype: `export { fromJsonSchema } from "@oh-my-pi/omptype";`, +}; + +/** Minified and gzipped bytes by entry. */ +export type SizeResults = Record; + +const results: SizeResults = {}; +for (const [name, contents] of Object.entries(entries)) { + const output = await build({ + stdin: { contents, resolveDir: src, loader: "ts" }, + bundle: true, + minify: true, + format: "esm", + platform: "browser", + write: false, + logLevel: "silent", + }); + const code = output.outputFiles[0].contents; + results[name] = { minified: code.length, gzip: gzipSync(code, { level: 9 }).length }; +} + +console.log("| library | minified | gzip |"); +console.log("|---|---:|---:|"); +for (const [name, r] of Object.entries(results)) { + console.log(`| ${name} | ${(r.minified / 1024).toFixed(1)} KB | ${(r.gzip / 1024).toFixed(1)} KB |`); +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); diff --git a/packages/schema/test/bench/speed.ts b/packages/schema/test/bench/speed.ts new file mode 100644 index 000000000..ba2f17b23 --- /dev/null +++ b/packages/schema/test/bench/speed.ts @@ -0,0 +1,79 @@ +// Hot validation speed: ns per check for each case and candidate, after warm-up. Each round times every +// candidate once, in a rotated order; the result is the median of the rounds. +// Run: node test/bench/speed.ts [--json ] (add --disallow-code-generation-from-strings for no eval) + +import { writeFileSync } from "node:fs"; +import { candidates } from "./candidates.ts"; +import { cases } from "./cases.ts"; + +const ROUNDS = Number(process.env.ROUNDS ?? 15); +const ROUND_MS = Number(process.env.ROUND_MS ?? 30); +let sink = 0; + +function time(check: (v: unknown) => boolean, value: unknown, n: number): number { + const start = process.hrtime.bigint(); + for (let i = 0; i < n; i++) if (check(value)) sink++; + return Number(process.hrtime.bigint() - start) / n; +} + +function median(values: number[]): number { + return [...values].sort((a, b) => a - b)[values.length >> 1]; +} + +/** + * Median ns per check, by case name, input ("valid"/"invalid") and candidate name; null where the candidate cannot + * compile the schema (Ajv without eval). + */ +export type SpeedResults = Record>>; + +const results: SpeedResults = {}; +for (const c of cases) { + results[c.name] = { valid: {}, invalid: {} }; + for (const input of ["valid", "invalid"] as const) { + const value = c[input]; + const runs: { + candidate: (typeof candidates)[number]; + check: (v: unknown) => boolean; + n: number; + samples: number[]; + }[] = []; + for (const candidate of candidates) { + let check: (v: unknown) => boolean; + try { + check = candidate.prepare(c.schema, { draft: "2020-12" }).check; + } catch (error) { + if (!(error instanceof EvalError)) throw error; + results[c.name][input][candidate.name] = null; + continue; + } + if (check(value) !== (input === "valid")) throw new Error(`${candidate.name} gets ${c.name} (${input}) wrong`); + let n = 1000; + while (time(check, value, n) * n < ROUND_MS * 1e6 && n < 1e8) n *= 2; + runs.push({ candidate, check, n, samples: [] }); + } + for (let round = 0; round < ROUNDS; round++) { + for (let i = 0; i < runs.length; i++) { + const run = runs[(i + round) % runs.length]; + run.samples.push(time(run.check, value, run.n)); + } + } + for (const run of runs) results[c.name][input][run.candidate.name] = median(run.samples); + } +} + +const names = candidates.map((c) => c.name); +console.log(`| case | input | ${names.join(" | ")} |`); +console.log(`|---|---|${names.map(() => "---:").join("|")}|`); +for (const [name, inputs] of Object.entries(results)) { + for (const [input, ns] of Object.entries(inputs)) { + console.log(`| ${name} | ${input} | ${names.map((n) => format(ns[n])).join(" | ")} |`); + } +} +const out = process.argv.indexOf("--json"); +if (out >= 0) writeFileSync(process.argv[out + 1], JSON.stringify(results, null, "\t")); +if (sink < 0) console.log(sink); + +function format(ns: number | null): string { + if (ns === null) return "fails"; + return ns >= 1000 ? `${(ns / 1000).toFixed(ns >= 10000 ? 0 : 1)} µs` : `${ns.toFixed(0)} ns`; +} diff --git a/packages/schema/test/bench/zod.ts b/packages/schema/test/bench/zod.ts new file mode 100644 index 000000000..c0d4d8371 --- /dev/null +++ b/packages/schema/test/bench/zod.ts @@ -0,0 +1,187 @@ +// Zod in the comparison. Zod is not a JSON Schema validator: the speed cases are written here as the native Zod +// schemas a developer would write (z.strictObject for closed objects, z.discriminatedUnion for tagged unions, z.lazy +// for recursion), and the corpus schemas are built into the same native calls by `nativeZod`. Only the conformance run +// goes through z.fromJSONSchema, Zod's (semi-experimental) JSON Schema import. + +import * as z from "zod"; +import type { JsonSchema } from "../../src/index.ts"; +import { cases } from "./cases.ts"; + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +const block = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("text"), text: z.string() }), + z.strictObject({ + kind: z.literal("image"), + src: z.string(), + width: z.number().int().min(0), + height: z.number().int().min(0), + alt: z.string().optional(), + }), + z.strictObject({ kind: z.literal("code"), lang: z.enum(["ts", "js", "py"]), source: z.string() }), +]); + +const id = z.number().int().min(0); + +const range = z.strictObject({ start: z.number().int().min(1), end: z.number().int().min(1) }); +const replace = z.strictObject({ oldText: z.string(), newText: z.string() }); +const lineEdit = z.strictObject({ range, text: z.string() }); + +const richText = z.strictObject({ + type: z.enum(["text", "mention"]), + content: z.string().max(2000), + annotations: z.strictObject({ bold: z.boolean(), italic: z.boolean(), color: z.string() }).optional(), +}); + +type PageBlock = { + type: "paragraph" | "heading_1" | "heading_2" | "bulleted_list_item" | "to_do"; + rich_text: z.infer[]; + checked?: boolean; + children?: PageBlock[]; +}; + +const pageBlock: z.ZodType = z.lazy(() => + z.strictObject({ + type: z.enum(["paragraph", "heading_1", "heading_2", "bulleted_list_item", "to_do"]), + rich_text: z.array(richText), + checked: z.boolean().optional(), + children: z.array(pageBlock).optional(), + }), +); + +/** The speed cases as native Zod schemas, by case name (the JSON Schema cases in cases.ts, one for one). */ +const native: Record = { + "small closed object": z.strictObject({ text: z.string() }), + "nested object with tagged unions": z.strictObject({ + id: z.string().regex(uuid), + title: z.string().min(1).max(200), + tags: z.array(z.string()).max(16), + author: z.strictObject({ name: z.string(), email: z.string().optional() }), + blocks: z.array(block), + score: z.number().min(0).max(1).optional(), + draft: z.boolean().optional(), + }), + "message union of 5": z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("hello"), + protocol: z.number().int().min(0), + incarnation: z.string().regex(uuid), + calls: z.number().int().min(1), + }), + z.strictObject({ + kind: z.literal("hold"), + hold: id, + type: z.string(), + version: z.number().int().min(1), + fingerprint: z.string().regex(/^[0-9a-f]{16}$/), + params: z.unknown(), + address: z.strictObject({ kind: z.string(), params: z.unknown() }), + chain: z.array(z.string().regex(uuid)).max(64).optional(), + }), + z.strictObject({ kind: z.literal("release"), hold: id }), + z.strictObject({ kind: z.literal("call"), call: id, hold: id, method: id, args: z.unknown().optional() }), + z.strictObject({ kind: z.literal("abort"), call: id }), + ]), + "tool: read": z.object({ path: z.string(), offset: z.number().optional(), limit: z.number().optional() }), + "tool: edit": z.object({ path: z.string(), edits: z.array(z.object({ oldText: z.string(), newText: z.string() })) }), + "tool: bash": z.object({ command: z.string(), timeout: z.number().optional() }), + "MCP create_issue (pattern, anyOf, enum)": z.strictObject({ + owner: z + .string() + .min(1) + .regex(/^[A-Za-z0-9-]+$/), + repo: z + .string() + .min(1) + .regex(/^[A-Za-z0-9._-]+$/), + title: z.string().min(1), + body: z.string().optional(), + labels: z.array(z.string()).optional(), + assignees: z.array(z.string()).optional(), + milestone: z.number().nullable().optional(), + state: z.enum(["open", "closed"]).optional(), + }), + "MCP edit_file ($ref, anyOf)": z.strictObject({ + path: z.string().regex(/^(\/|\.\/|[A-Za-z]:\\)?[^\u0000]+$/), + edits: z.array(z.union([replace, lineEdit])).min(1), + dryRun: z.boolean().optional(), + }), + "MCP create_page (recursive $ref, oneOf)": z.strictObject({ + parent: z.union([ + z.strictObject({ page_id: z.string().regex(uuid) }), + z.strictObject({ database_id: z.string().regex(uuid) }), + ]), + title: z.string().min(1), + due: z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/) + .optional(), + children: z.array(pageBlock).max(100).optional(), + }), +}; + +/** The native Zod schema of each speed case's JSON Schema. */ +export const nativeCases: ReadonlyMap = new Map( + cases.map((c) => { + const schema = native[c.name]; + if (schema === undefined) throw new Error(`No native Zod schema for the case ${c.name}`); + return [c.schema, schema]; + }), +); + +type Json = Record; + +/** + * The native Zod schema a developer would write for a corpus schema (corpus.ts): objects (closed ones strict), + * strings with bounds, patterns and enums, bounded integers, numbers, booleans, string arrays, descriptions, and + * $defs reused through their constant. Throws for any other shape. + */ +export function nativeZod(schema: Json, root: Json = schema): z.ZodType { + if (typeof schema.$ref === "string") { + const name = schema.$ref.replace("#/$defs/", ""); + return nativeZod((root.$defs as Record)[name], root); + } + const built = nativeType(schema, root); + return typeof schema.description === "string" ? built.describe(schema.description) : built; +} + +function nativeType(schema: Json, root: Json): z.ZodType { + switch (schema.type) { + case "object": { + const required = new Set(schema.required as string[]); + const shape: Record = {}; + for (const [key, value] of Object.entries(schema.properties as Record)) { + const property = nativeZod(value, root); + shape[key] = required.has(key) ? property : property.optional(); + } + return schema.additionalProperties === false ? z.strictObject(shape) : z.object(shape); + } + case "string": { + if (Array.isArray(schema.enum)) return z.enum(schema.enum as [string, ...string[]]); + let string = z.string(); + if (typeof schema.minLength === "number") string = string.min(schema.minLength); + if (typeof schema.maxLength === "number") string = string.max(schema.maxLength); + if (typeof schema.pattern === "string") string = string.regex(new RegExp(schema.pattern)); + return string; + } + case "integer": + return z + .number() + .int() + .min(schema.minimum as number) + .max(schema.maximum as number); + case "number": + return z.number(); + case "boolean": + return z.boolean(); + case "array": + return z.array(z.string()).max(schema.maxItems as number); + default: + throw new Error(`No native Zod builder for ${JSON.stringify(schema).slice(0, 80)}`); + } +} + +/** z.fromJSONSchema with the draft given; it accepts only local $refs, so remote documents are not passed. */ +export function zodFromJsonSchema(schema: JsonSchema, draft: "2020-12" | "7"): z.ZodType { + return z.fromJSONSchema(schema as never, { defaultTarget: draft === "7" ? "draft-7" : "draft-2020-12" }); +} diff --git a/packages/schema/test/builders.test.ts b/packages/schema/test/builders.test.ts new file mode 100644 index 000000000..2d13886c7 --- /dev/null +++ b/packages/schema/test/builders.test.ts @@ -0,0 +1,104 @@ +// The builders: the JSON Schema they produce, the static types they carry, and how they compile. + +import { describe, expect, expectTypeOf, it } from "vitest"; +import { + arr, + bool, + bytes, + compile, + int, + lit, + map, + nul, + nullable, + num, + obj, + oneOf, + opt, + type Static, + str, + tuple, + union, + unknown, +} from "../src/index.ts"; + +describe("builders", () => { + it("produce plain JSON Schema", () => { + const schema = obj({ + path: str({ description: "File path", minLength: 1 }), + offset: opt(int({ minimum: 1 })), + mode: oneOf(["read", "write"]), + }); + expect(JSON.parse(JSON.stringify(schema))).toEqual({ + type: "object", + properties: { + path: { type: "string", description: "File path", minLength: 1 }, + offset: { type: "integer", minimum: 1 }, + mode: { enum: ["read", "write"] }, + }, + required: ["path", "mode"], + additionalProperties: false, + }); + }); + + it("produce every leaf", () => { + expect([str(), num(), int(), bool(), nul(), unknown(), lit(3), bytes({ maxLength: 8 })]).toEqual([ + { type: "string" }, + { type: "number" }, + { type: "integer" }, + { type: "boolean" }, + { type: "null" }, + {}, + { const: 3 }, + { type: "bytes", maxLength: 8 }, + ]); + expect(arr(str(), { maxItems: 2 })).toEqual({ type: "array", items: { type: "string" }, maxItems: 2 }); + expect(map(num())).toEqual({ type: "object", additionalProperties: { type: "number" } }); + expect(tuple([str(), num()])).toEqual({ + type: "array", + prefixItems: [{ type: "string" }, { type: "number" }], + items: false, + minItems: 2, + }); + expect(nullable(str())).toEqual({ anyOf: [{ type: "string" }, { type: "null" }] }); + expect(obj({ a: str() }, { open: true })).toEqual({ + type: "object", + properties: { a: { type: "string" } }, + required: ["a"], + }); + }); + + it("carry the static type of valid values", () => { + const message = union([ + obj({ kind: lit("text"), text: str() }), + obj({ kind: lit("image"), url: str(), alt: opt(str()) }), + ]); + expectTypeOf>().toEqualTypeOf< + { kind: "text"; text: string } | { kind: "image"; url: string; alt?: string } + >(); + const pair = tuple([str(), arr(int())]); + expectTypeOf>().toEqualTypeOf<[string, number[]]>(); + expectTypeOf>>>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + }); + + it("narrow values through a compiled check", () => { + const point = obj({ x: num(), y: num(), label: opt(str()) }); + const value: unknown = { x: 1, y: 2 }; + const { check } = compile(point); + if (!check(value)) throw new Error("expected a point"); + expectTypeOf(value).toEqualTypeOf<{ x: number; y: number; label?: string }>(); + expect(value.x + value.y).toBe(3); + }); + + it("compile into checks with the declared semantics", () => { + const validator = compile(obj({ name: str({ minLength: 1 }), tags: opt(arr(str(), { uniqueItems: true })) })); + expect(validator.check({ name: "a" })).toBe(true); + expect(validator.check({ name: "a", tags: ["x", "y"] })).toBe(true); + expect(validator.errors({ name: "", tags: ["x", "x"], other: 1 })).toEqual([ + { path: "/name", keyword: "minLength", message: "expected at least 1 characters" }, + { path: "/tags", keyword: "uniqueItems", message: "expected unique items" }, + { path: "/other", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); +}); diff --git a/packages/schema/test/coerce-typebox.test.ts b/packages/schema/test/coerce-typebox.test.ts new file mode 100644 index 000000000..f9a83140d --- /dev/null +++ b/packages/schema/test/coerce-typebox.test.ts @@ -0,0 +1,1344 @@ +// biome-ignore-all lint/suspicious/noTemplateCurlyInString: TypeBox template literal syntax, as in the original tests. +// biome-ignore-all lint/correctness/noPrecisionLoss: the original tests' literal, kept as written. +// TypeBox 1.3.27's Value.Convert tests (test/typebox/runtime/value/convert, MIT, (c) Haydn Paterson), ported to the +// conversion pass of pi-schema's coercion. Schemas are still built with TypeBox: its schemas are JSON Schema, and the +// port reads only their JSON structure, never TypeBox's kind marks. Run with vitest. + +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { Coercer } from "../src/coerce.ts"; +import { Compiler } from "../src/compiler.ts"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; + +/** Pass 2 alone, as TypeBox's Value.Convert(schema, value). */ +function convert(schema: unknown, value: unknown): unknown { + return new Coercer(new Compiler(schema as object, { formats }, false), schema).convertOnly(value); +} + +const Value = { + Convert: convert, + Check: (schema: unknown, value: unknown) => compile(schema as object, { formats }).check(value), +}; +const Convert = convert; +const Assert = { + IsEqual: (actual: unknown, expected: unknown) => expect(actual).toEqual(expected), + IsTrue: (value: boolean) => expect(value).toBe(true), + IsFalse: (value: boolean) => expect(value).toBe(false), +}; + +describe("Value.Convert.Any", () => { + it("Should Convert 1", () => { + const T = Type.Any(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 2", () => { + const T = Type.Any(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, undefined); + }); + it("Should Convert 3", () => { + const T = Type.Any(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 4", () => { + const T = Type.Any(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 5", () => { + const T = Type.Any(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Any(); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Any(); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsEqual(R, [1, 2, 3]); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 8", () => { + const S = Symbol("x-symbol"); + const T = Type.Symbol(); + const R = Value.Convert(T, S); + Assert.IsEqual(R, S); + }); +}); + +describe("Value.Convert.Array", () => { + it("Should Convert 1", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, [1, 3.14, 1, 3.14, 1, 0, 1, 0, "hello"]); + }); + it("Should Convert 2", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, [true, 3.14, true, "3.14", true, false, true, false, "hello"]); + }); + it("Should Convert 3", () => { + const T = Type.Array(Type.String()); + const R = Value.Convert(T, [1, 3.14, "1", "3.14", true, false, "true", "false", "hello"]); + Assert.IsEqual(R, ["1", "3.14", "1", "3.14", "true", "false", "true", "false", "hello"]); + }); + // ------------------------------------------------------------------ + // Added support for Array coercion + // + // https://github.com/sinclairzx81/typebox/issues/1469 + // ------------------------------------------------------------------ + it("Should Convert 4", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 5", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + it("Should Convert 6", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, [42]); + }); + it("Should Convert 7", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, true); + Assert.IsEqual(R, [true]); + }); + it("Should Convert 8", () => { + const T = Type.Array(Type.Null()); + const R = Value.Convert(T, null); + Assert.IsEqual(R, [null]); + }); + it("Should Convert 9", () => { + const T = Type.Array(Type.Unknown()); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, [undefined]); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 10", () => { + const T = Type.Array(Type.BigInt()); + const R = Value.Convert(T, 1n); + Assert.IsEqual(R, [1n]); + }); + it("Should Convert 11", () => { + const T = Type.Array(Type.Object({})); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, [{}]); + }); + it("Should Convert 12", () => { + const T = Type.Array(Type.Object({ x: Type.Number() })); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, [{ x: 1 }]); + }); + it("Should Convert 13", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, [42]); + }); + it("Should Convert 14", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, [true]); + }); + it("Should Convert 15", () => { + const T = Type.Array(Type.Boolean()); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, [false]); + }); + it("Should Convert 16", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, ["hello"]); + }); + it("Should Convert 17", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, [1]); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 18", () => { + const T = Type.Array(Type.Array(Type.Number())); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 19", () => { + const T = Type.Array(Type.Number()); + const R = Value.Convert(T, { 0: 1, length: 1 }); + Assert.IsEqual(R, [{ 0: 1, length: 1 }]); + }); + it("Should Convert 20", () => { + const T = Type.Array(Type.Array(Type.Number())); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + // ------------------------------------------------------------------ + // Union + // ------------------------------------------------------------------ + it("Should Convert 21", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, 1); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 22", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, "hello"); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 23", () => { + const T = Type.Array(Type.Union([Type.Number(), Type.String()])); + const R = Value.Convert(T, [1, "hello", true, "3.14"]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 24", () => { + const T = Type.Union([Type.Array(Type.Number()), Type.String()]); + const R = Value.Convert(T, "1"); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 25", () => { + const T = Type.Array(Type.Union([Type.Array(Type.Number()), Type.String()])); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + // ------------------------------------------------------------------ + // Embedded Intersect + // ------------------------------------------------------------------ + it("Should Convert 26", () => { + const T = Type.Array(Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })])); + const R = Value.Convert(T, { x: 1, y: 2 }); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 27", () => { + const T = Type.Array(Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })])); + const R = Value.Convert(T, [ + { x: "1", y: "2" }, + { x: "3", y: "4" }, + ]); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 28", () => { + const T = Type.Intersect([Type.Array(Type.Number()), Type.Array(Type.Number())]); + const R = Value.Convert(T, 42); + Assert.IsTrue(Value.Check(T, R)); + }); + it("Should Convert 29", () => { + const T = Type.Intersect([Type.Array(Type.Number()), Type.Object({ x: Type.Number() })]); + const R = Value.Convert(T, 42); + Assert.IsFalse(Value.Check(T, R)); + }); +}); + +describe("Value.Convert.Boolean", () => { + it("Should Convert 1", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, true); + }); + it("Should Convert 2", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, "3.14"); + }); + it("Should Convert 3", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, true); + }); + it("Should Convert 4", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, false); + }); + it("Should Convert 5", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 7", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, 1.1); + Assert.IsEqual(R, 1.1); + }); + it("Should Convert 8", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 9", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, false); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 10", () => { + const value = "hello"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, "hello"); + }); + it("Should Convert 11", () => { + const value = "true"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 12", () => { + const value = "TRUE"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 13", () => { + const value = "false"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 14", () => { + const value = "0"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 15", () => { + const value = "1"; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 16", () => { + const value = "0"; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 17", () => { + const value = "1"; + const result = Value.Convert(Type.Boolean({ default: false }), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 18", () => { + const value = "2"; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, "2"); + }); + it("Should Convert 19", () => { + const value = 0; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 20", () => { + const value = 1n; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 21", () => { + const value = 1; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 22", () => { + const value = 2; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, 2); + }); + it("Should Convert 23", () => { + const value = 0; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 24", () => { + const value = 1; + const result = Value.Convert(Type.Boolean({ default: false }), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 25", () => { + const value = 2; + const result = Value.Convert(Type.Boolean({ default: true }), value); + Assert.IsEqual(result, 2); + }); + it("Should Convert 26", () => { + const value = true; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, true); + }); + it("Should Convert 27", () => { + const value = false; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, false); + }); + it("Should Convert 28", () => { + const value = {}; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, {}); + }); + it("Should Convert 29", () => { + const value = [] as any[]; + const result = Value.Convert(Type.Boolean(), value); + Assert.IsEqual(result, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 17", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, false); + }); + it("Should Convert 18", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(1)); + Assert.IsEqual(R, true); + }); + it("Should Convert 19", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, BigInt(2)); + Assert.IsEqual(R, BigInt(2)); + }); + it("Should Convert 20", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 21", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, false); + }); + it("Should Convert 22", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, false); + }); + it("Should Convert 23", () => { + const T = Type.Boolean(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, false); + }); +}); + +describe("Value.Convert.Enum", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Enum([1, 2, "hello"]), + }); + const R0 = Value.Convert(T, { x: "1" }); + const R1 = Value.Convert(T, { x: "2" }); + const R2 = Value.Convert(T, { x: "hello" }); + const R3 = Value.Convert(T, { x: "world" }); + Assert.IsEqual(R0, { x: 1 }); + Assert.IsEqual(R1, { x: 2 }); + Assert.IsEqual(R2, { x: "hello" }); + Assert.IsEqual(R3, { x: "world" }); + }); +}); + +describe("Value.Convert.Integer", () => { + it("Should Convert 1", () => { + const T = Type.Integer(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3); + }); + it("Should Convert 2", () => { + const T = Type.Integer(); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, 42); + }); + it("Should Convert 3", () => { + const T = Type.Integer(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 4", () => { + const T = Type.Integer(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 5", () => { + const T = Type.Integer(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const value = "hello"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, "hello"); + }); + it("Should Convert 7", () => { + const value = "3.14"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 3); + }); + it("Should Convert 8", () => { + const value = "-0"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, -0); + }); + it("Should Convert 9", () => { + const value = "-100"; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, -100); + }); + it("Should Convert 10", () => { + const value = 42; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 42); + }); + it("Should Convert 11", () => { + const value = true; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 1); + }); + it("Should Convert 12", () => { + const value = false; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, 0); + }); + it("Should Convert 13", () => { + const value = {}; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, {}); + }); + it("Should Convert 14", () => { + const value = [] as any[]; + const result = Value.Convert(Type.Integer(), value); + Assert.IsEqual(result, []); + }); + // ---------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/1147 + // ---------------------------------------------------------- + it("Should Convert 15", () => { + const N = 1738213389080; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, N); + }); + it("Should Convert 16", () => { + const N = 1738213389080.5555; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); + it("Should Convert 17", () => { + const N = "1738213389080"; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); + it("Should Convert 18", () => { + const N = "1738213389080.555"; + const R = Value.Convert(Type.Integer(), N); + Assert.IsEqual(R, 1738213389080); + }); +}); + +describe("Value.Convert.Intersect", () => { + it("Should Convert intersected objects", () => { + const T = Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })]); + const R = Value.Convert(T, { x: "1", y: "2" }); + Assert.IsEqual(R, { x: 1, y: 2 }); + }); + // ---------------------------------------------------------------- + // Intersection Complex + // ---------------------------------------------------------------- + it("Should complex intersect 1", () => { + const T = Type.Intersect([Type.Number(), Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() })]); + const R = Value.Convert(T, { x: "1", y: "2" }); + Assert.IsEqual(R, { x: 1, y: 2 }); + }); + it("Should complex intersect 2", () => { + const T = Type.Intersect([Type.Object({ x: Type.Number() }), Type.Object({ y: Type.Number() }), Type.Number()]); + const R = Value.Convert(T, { x: "3", y: "4" }); + Assert.IsEqual(R, { x: 3, y: 4 }); + }); + // Differs from TypeBox: allOf converts with each member in turn; TypeBox first merges an intersection into one object and drops the number. + it("Should complex intersect 3", () => { + const T = Type.Intersect([ + Type.Number(), // lost in Evaluate + Type.Object({ x: Type.Number() }), + Type.Object({ y: Type.Number() }), + ]); + const R = Value.Convert(T, "123"); + Assert.IsEqual(R, 123); + }); +}); + +describe("Value.Convert.Literal", () => { + it("LiteralString: Should convert 1", () => { + const T = Type.Literal("1"); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, "1"); + }); + it("LiteralString: Should convert 2", () => { + const T = Type.Literal("1"); + const R = Value.Convert(T, 2); + Assert.IsEqual(R, 2); + }); + it("LiteralString: Should convert 3", () => { + const T = Type.Literal("true"); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + // ------------------------------------------------------------------ + // LiteralNumber + // ------------------------------------------------------------------ + it("LiteralNumber: Should convert 1", () => { + const T = Type.Literal(3.14); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("LiteralNumber: Should convert 2", () => { + const T = Type.Literal(3.14); + const R = Value.Convert(T, "3.15"); + Assert.IsEqual(R, "3.15"); + }); + it("LiteralNumber: Should convert 3", () => { + const T = Type.Literal(1); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("LiteralNumber: Should convert 4", () => { + const T = Type.Literal(0); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("LiteralNumber: Should convert 5", () => { + const T = Type.Literal(2); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + // ------------------------------------------------------------------ + // LiteralBoolean + // ------------------------------------------------------------------ + it("LiteralBoolean: Should convert 1", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + it("LiteralBoolean: Should convert 2", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 3", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 4", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, false); + }); + it("LiteralBoolean: Should convert 5", () => { + const T = Type.Literal(true); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, true); + }); + it("LiteralBoolean: Should convert 6", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, false); + }); + it("LiteralBoolean: Should convert 7", () => { + const T = Type.Literal(false); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); + // ------------------------------------------------------------------ + // LiteralBigInt + // ------------------------------------------------------------------ + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 1", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100n); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 2", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 3", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 4", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 5", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 6", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 7", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); + // ------------------------------------------------------------------ + // LiteralBigInt + // ------------------------------------------------------------------ + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 1", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100n); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 2", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, 100); + Assert.IsEqual(R, 100n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 3", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 4", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 5", () => { + const T = Type.Literal(1n); + const R = Value.Convert(T, "1"); + Assert.IsEqual(R, 1n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 6", () => { + const T = Type.Literal(0n); + const R = Value.Convert(T, "0"); + Assert.IsEqual(R, 0n); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("LiteralBigInt: Should convert 7", () => { + const T = Type.Literal(100n); + const R = Value.Convert(T, "2"); + Assert.IsEqual(R, "2"); + }); +}); + +describe("Value.Convert.Never", () => { + it("Should not 1", () => { + const T = Type.Never(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should not 2", () => { + const T = Type.Never(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should not 3", () => { + const T = Type.Never(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, "true"); + }); +}); + +describe("Value.Convert.Null", () => { + it("Should Convert 1", () => { + const T = Type.Null(); + const R = Value.Convert(T, "null"); + Assert.IsEqual(R, null); + }); + it("Should Convert 2", () => { + const T = Type.Null(); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 3", () => { + const T = Type.Null(); + const R = Value.Convert(T, "nil"); + Assert.IsEqual(R, "nil"); + }); + it("Should Convert 4", () => { + const T = Type.Null(); + const R = Value.Convert(T, "undefined"); + Assert.IsEqual(R, null); + }); + it("Should Convert 5", () => { + const T = Type.Null(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 6", () => { + const T = Type.Null(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Null(); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, null); + }); + it("Should Convert 8", () => { + const T = Type.Null(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, null); + }); + it("Should Convert 9", () => { + const T = Type.Null(); + const R = Value.Convert(T, ""); + Assert.IsEqual(R, null); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 10", () => { + const T = Type.Null(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, null); + }); + it("Should Convert 11", () => { + const T = Type.Null(); + const R = Value.Convert(T, BigInt(1)); + Assert.IsEqual(R, BigInt(1)); + }); + it("Should Convert 12", () => { + const T = Type.Null(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, null); + }); + it("Should Convert 13", () => { + const T = Type.Null(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 14", () => { + const T = Type.Null(); + const R = Value.Convert(T, 0); + Assert.IsEqual(R, null); + }); + it("Should Convert 15", () => { + const T = Type.Null(); + const R = Value.Convert(T, 1); + Assert.IsEqual(R, 1); + }); + it("Should Convert 16", () => { + const T = Type.Null(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 17", () => { + const T = Type.Null(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, null); + }); + // Skipped: the schema or the value has no JSON form. + it.skip("Should Convert 18", () => { + const T = Type.Null(); + const S = Symbol("hello"); + const R = Value.Convert(T, S); + Assert.IsEqual(R, S); + }); +}); + +describe("Value.Convert.Number", () => { + it("Should Convert 1", () => { + const T = Type.Number(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 2", () => { + const T = Type.Number(); + const R = Value.Convert(T, "42"); + Assert.IsEqual(R, 42); + }); + it("Should Convert 3", () => { + const T = Type.Number(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 4", () => { + const T = Type.Number(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 5", () => { + const T = Type.Number(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, 3.14); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const T = Type.Number(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 7", () => { + const T = Type.Number(); + const R = Value.Convert(T, "3.14"); + Assert.IsEqual(R, 3.14); + }); + it("Should Convert 8", () => { + const T = Type.Number(); + const R = Value.Convert(T, "-0"); + Assert.IsEqual(R, -0); + }); + it("Should Convert 9", () => { + const T = Type.Number(); + const R = Value.Convert(T, "-100"); + Assert.IsEqual(R, -100); + }); + it("Should Convert 10", () => { + const T = Type.Number(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 11", () => { + const T = Type.Number(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, 1); + }); + it("Should Convert 12", () => { + const T = Type.Number(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, 0); + }); + it("Should Convert 13", () => { + const T = Type.Number(); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 14", () => { + const T = Type.Number(); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 10", () => { + const T = Type.Number(); + const R = Value.Convert(T, BigInt(0)); + Assert.IsEqual(R, 0); + }); + it("Should Convert 11", () => { + const T = Type.Number(); + const R = Value.Convert(T, BigInt(10000000000000000000000000000000000000000000000000000)); + Assert.IsEqual(R, BigInt(10000000000000000000000000000000000000000000000000000)); + }); + it("Should Convert 12", () => { + const T = Type.Number(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, 0); + }); + it("Should Convert 13", () => { + const T = Type.Number(); + const R = Value.Convert(T, "false"); + Assert.IsEqual(R, 0); + }); + it("Should Convert 14", () => { + const T = Type.Number(); + const R = Value.Convert(T, "true"); + Assert.IsEqual(R, 1); + }); + it("Should Convert 15", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123"); + Assert.IsEqual(R, 123); + }); + it("Should Convert 16", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123.456"); + Assert.IsEqual(R, 123.456); + }); + it("Should Convert 17", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123.456"); + Assert.IsEqual(R, 123.456); + }); + it("Should Convert 18", () => { + const T = Type.Number(); + const R = Value.Convert(T, "123n"); + Assert.IsEqual(R, 123); + }); + it("Should Convert 19", () => { + const T = Type.Number(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, 0); + }); + it("Should Convert 20", () => { + const T = Type.Number(); + const R = Value.Convert(T, `90071992547409910n`); // trailing 0 too large + Assert.IsEqual(R, "90071992547409910n"); + }); +}); + +describe("Value.Convert.Object", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Number(), + y: Type.Boolean(), + z: Type.Boolean(), + }); + const R = Value.Convert(T, { x: "42", y: "true", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: true, z: "hello" }); + }); + it("Should Convert 2", () => { + const T = Type.Object({ + x: Type.Number(), + y: Type.Boolean(), + }); + const R = Value.Convert(T, { x: "42", y: "true", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: true, z: "hello" }); + }); + it("Should Convert 3", () => { + const T = Type.Object({ x: Type.Number() }); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 4", () => { + const T = Type.Object( + { x: Type.Number() }, + { + additionalProperties: Type.Null(), + }, + ); + const R = Value.Convert(T, { x: "1", y: 0 }); + Assert.IsEqual(R, { x: 1, y: null }); + }); + // ------------------------------------------------------------------ + // IsOptionalUndefined + // + // https://github.com/sinclairzx81/typebox/issues/1336#issuecomment-3312808962 + // ------------------------------------------------------------------ + it("Should Convert 5", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 6", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: "1" }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Object({ x: Type.Optional(Type.Number()) }); + const R = Value.Convert(T, { x: undefined }); + Assert.IsEqual(R, { x: undefined }); + }); +}); + +describe("Value.Convert.Record", () => { + it("Should Convert 1", () => { + const T = Type.Record(Type.String(), Type.Number()); + const R = Value.Convert(T, { x: "42", y: "24", z: "hello" }); + Assert.IsEqual(R, { x: 42, y: 24, z: "hello" }); + }); + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/930 + // ---------------------------------------------------------------- + it("Should Convert 2", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 3", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 4", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 5", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 6", () => { + const T = Type.Union([Type.Null(), Type.Record(Type.Number(), Type.Any())]); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Union([Type.Record(Type.Number(), Type.Any()), Type.Null()]); + const R = Value.Convert(T, "NULL"); + Assert.IsEqual(R, null); + }); + it("Should Convert 8", () => { + const T = Type.Record(Type.Number(), Type.Number(), { + additionalProperties: Type.Null(), + }); + const R = Value.Convert(T, { "0": "1", x: 0 }); + Assert.IsEqual(R, { "0": 1, x: null }); + }); +}); + +describe("Value.Convert.String", () => { + it("Should Convert 1", () => { + const T = Type.String(); + const R = Value.Convert(T, 3.14); + Assert.IsEqual(R, "3.14"); + }); + it("Should Convert 2", () => { + const T = Type.String(); + const R = Value.Convert(T, 3); + Assert.IsEqual(R, "3"); + }); + it("Should Convert 3", () => { + const T = Type.String(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + it("Should Convert 4", () => { + const T = Type.String(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, "false"); + }); + it("Should Convert 5", () => { + const T = Type.String(); + const R = Value.Convert(T, BigInt(12345)); + Assert.IsEqual(R, "12345"); + }); + // ---------------------------------------------------------- + // Casts + // ---------------------------------------------------------- + it("Should Convert 6", () => { + const T = Type.String(); + const R = Value.Convert(T, "hello"); + Assert.IsEqual(R, "hello"); + }); + it("Should Convert 7", () => { + const T = Type.String(); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, "42"); + }); + it("Should Convert 8", () => { + const T = Type.String(); + const R = Value.Convert(T, 42n); + Assert.IsEqual(R, "42"); + }); + it("Should Convert 9", () => { + const T = Type.String(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, "true"); + }); + it("Should Convert 10", () => { + const T = Type.String(); + const R = Value.Convert(T, false); + Assert.IsEqual(R, "false"); + }); + it("Should Convert 11", () => { + const T = Type.String(); + const R = Value.Convert(T, {}); + Assert.IsEqual(R, {}); + }); + it("Should Convert 12", () => { + const T = Type.String(); + const R = Value.Convert(T, []); + Assert.IsEqual(R, []); + }); + // ------------------------------------------------------------------ + // Coverage + // ------------------------------------------------------------------ + it("Should Convert 13", () => { + const T = Type.String(); + const R = Value.Convert(T, null); + Assert.IsEqual(R, "null"); + }); + it("Should Convert 14", () => { + const T = Type.String(); + const R = Value.Convert(T, undefined); + Assert.IsEqual(R, ""); + }); +}); + +describe("Value.Convert.TemplateLiteral", () => { + // Differs from TypeBox: a template literal is a string with a pattern in JSON Schema; TypeBox converts a finite one as a union of literals. + it("Should Convert 1", () => { + const T = Type.TemplateLiteral("${1 | 2 | 3}"); + const R0 = Value.Convert(T, 1); + const R1 = Value.Convert(T, 2); + const R2 = Value.Convert(T, 3); + const R3 = Value.Convert(T, 4); + Assert.IsEqual(R0, "1"); + Assert.IsEqual(R1, "2"); + Assert.IsEqual(R2, "3"); + Assert.IsEqual(R3, "4"); + }); + // ------------------------------------------------------------------ + // https://github.com/sinclairzx81/typebox/issues/1510 + // ------------------------------------------------------------------ + it("Should Convert 2", () => { + const T = Type.Object({ + test: Type.TemplateLiteral('${number}${"a"|"b"|"c"}'), + }); + const R0 = Value.Convert(T, { test: "123b" }); + Assert.IsEqual(R0, { test: "123b" }); + }); +}); + +describe("Value.Convert.Tuple", () => { + it("Should Convert 1", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1", "true"]); + Assert.IsEqual(R, [1, 1]); + }); + it("Should Convert 2", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1"]); + Assert.IsEqual(R, [1]); + }); + it("Should Convert 3", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, ["1", "2", "3"]); + Assert.IsEqual(R, [1, 2, "3"]); + }); + it("Should Convert 4", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, "not an array"); + Assert.IsEqual(R, "not an array"); + }); + it("Should Convert 5", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, 42); + Assert.IsEqual(R, 42); + }); + it("Should Convert 6", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, null); + Assert.IsEqual(R, null); + }); + it("Should Convert 7", () => { + const T = Type.Tuple([Type.Number(), Type.Number()]); + const R = Value.Convert(T, { 0: "1", 1: "2" }); + Assert.IsEqual(R, { 0: "1", 1: "2" }); + }); +}); + +describe("Value.Convert.Union", () => { + it("Should Convert 1", () => { + const T = Type.Object({ + x: Type.Union([Type.Number(), Type.Null()]), + }); + const R0 = Value.Convert(T, { x: "42" }); + const R1 = Value.Convert(T, { x: "null" }); + const R2 = Value.Convert(T, { x: "hello" }); + Assert.IsEqual(R0, { x: 42 }); + Assert.IsEqual(R1, { x: null }); + Assert.IsEqual(R2, { x: "hello" }); + }); + it("Should Convert 2", () => { + const T = Type.Object({ + x: Type.Union([Type.Boolean(), Type.Number()]), + }); + const R = Value.Convert(T, { x: "1" }); + Assert.IsEqual(R, { x: true }); + }); + + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/787 + // ---------------------------------------------------------------- + it("Should Convert 3", () => { + const T = Type.Intersect([ + Type.Union([Type.Object({ a: Type.Number() }), Type.Object({ b: Type.Number() })]), + Type.Object({ c: Type.Number() }), + ]); + const R0 = Convert(T, { a: "1", c: "2" }); + const R1 = Convert(T, { b: "1", c: "2" }); + const R2 = Convert(T, { a: "1", b: "2", c: "3" }); + Assert.IsEqual(R0, { a: 1, c: 2 }); + Assert.IsEqual(R1, { b: 1, c: 2 }); + Assert.IsEqual(R2, { a: 1, b: "2", c: 3 }); // note: matching on first + }); + // ---------------------------------------------------------------- + // https://github.com/sinclairzx81/typebox/issues/1295 + // ---------------------------------------------------------------- + it("Should guard against Array conversion in Object", () => { + const T = Type.Union([ + Type.Object({ + type: Type.Literal("A"), + values: Type.Union([Type.String(), Type.Number()]), + }), + Type.Object({ + type: Type.Literal("B"), + values: Type.String(), + }), + ]); + const converted = Value.Convert(T, [{ type: "A", values: 1 }]); + Assert.IsEqual(converted, [{ type: "A", values: 1 }]); + }); + it("Should guard against Array conversion in Record", () => { + const T = Type.Union([ + Type.Record(Type.String({ pattern: "^values$" }), Type.Union([Type.String(), Type.Number()])), + Type.Record(Type.String({ pattern: "^type$" }), Type.Union([Type.String(), Type.Number()])), + ]); + const converted = Value.Convert(T, [{ type: "A", values: 1 }]); + Assert.IsEqual(converted, [{ type: "A", values: 1 }]); + }); +}); + +describe("Value.Convert.Unknown", () => { + it("Should Convert 1", () => { + const V = null; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 2", () => { + const V = undefined; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 3", () => { + const V = "hello"; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 4", () => { + const V = 42; + const T = Type.Unknown(); + const R = Value.Convert(T, V); + Assert.IsEqual(R, V); + }); + it("Should Convert 5", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, true); + Assert.IsEqual(R, true); + }); + it("Should Convert 6", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, { x: 1 }); + Assert.IsEqual(R, { x: 1 }); + }); + it("Should Convert 7", () => { + const T = Type.Unknown(); + const R = Value.Convert(T, [1, 2, 3]); + Assert.IsEqual(R, [1, 2, 3]); + }); +}); diff --git a/packages/schema/test/coerce.test.ts b/packages/schema/test/coerce.test.ts new file mode 100644 index 000000000..4444f0cc9 --- /dev/null +++ b/packages/schema/test/coerce.test.ts @@ -0,0 +1,458 @@ +// compile(schema, { coerce: true }): coerce() and parse(), the three coercion passes (optional nulls, TypeBox's +// conversion rules, primitive coercion by type), their order where they conflict, and the guarantees around them. +// Ports pi-ai's tool argument coercion tests. + +import Type from "typebox"; +import { describe, expect, expectTypeOf, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { + arr, + compile, + int, + map, + nullable, + num, + obj, + opt, + str, + tuple, + type ValidationError, + type Validator, +} from "../src/index.ts"; + +function coerce(schema: object, value: unknown): unknown { + return compile(schema, { formats, coerce: true }).coerce(value); +} + +/** The coerced `value` property of `{ value }` under an object schema with one required property `value`. */ +function coerceValue(schema: unknown, value: unknown): unknown { + const root = { type: "object", properties: { value: schema }, required: ["value"] }; + return (coerce(root, { value }) as { value: unknown }).value; +} + +describe("compile options", () => { + it("adds coerce and parse only with coerce: true", () => { + const plain = compile(obj({ n: num() })); + expectTypeOf(plain).toEqualTypeOf>(); + expect("coerce" in plain).toBe(false); + expect("parse" in plain).toBe(false); + const coercing = compile(obj({ n: num() }), { coerce: true }); + expectTypeOf(coercing.parse).returns.toEqualTypeOf< + | { readonly ok: true; readonly value: { n: number } } + | { readonly ok: false; readonly errors: ValidationError[] } + >(); + expect(coercing.check({ n: 1 })).toBe(true); + }); +}); + +describe("coerce", () => { + it("never modifies the value", () => { + const value = { count: "42", tags: "a" }; + const schema = obj({ count: num(), tags: arr(str()) }); + expect(coerce(schema, value)).toEqual({ count: 42, tags: ["a"] }); + expect(value).toEqual({ count: "42", tags: "a" }); + }); + + it("returns the value itself when nothing changes", () => { + const value = { a: { b: [1, 2] }, c: "x" }; + const schema = obj({ a: obj({ b: arr(num()) }), c: str() }); + expect(coerce(schema, value)).toBe(value); + }); + + it("copies only the changed paths", () => { + const unchanged = { b: [1, 2] }; + const value = { a: unchanged, c: { d: "1" } }; + const schema = obj({ a: obj({ b: arr(num()) }), c: obj({ d: num() }) }); + const coerced = coerce(schema, value) as typeof value; + expect(coerced).toEqual({ a: { b: [1, 2] }, c: { d: 1 } }); + expect(coerced).not.toBe(value); + expect(coerced.a).toBe(unchanged); + expect(value.c).toEqual({ d: "1" }); + }); + + it("works on frozen schemas", () => { + const schema = Object.freeze({ + type: "object", + properties: Object.freeze({ n: Object.freeze({ type: "integer" }) }), + }); + expect(coerce(schema, { n: "7" })).toEqual({ n: 7 }); + }); + + it("still coerces when the Function constructor is unavailable", () => { + const original = globalThis.Function; + globalThis.Function = (() => { + throw new EvalError("Code generation from strings disallowed for this context"); + }) as unknown as FunctionConstructor; + try { + expect(coerce(obj({ count: num() }), { count: "42" })).toEqual({ count: 42 }); + } finally { + globalThis.Function = original; + } + }); + + it("leaves values it cannot coerce for the check to report", () => { + const validator = compile(obj({ count: num() }), { coerce: true }); + const coerced = validator.coerce({ count: "many" }); + expect(coerced).toEqual({ count: "many" }); + expect(validator.check(coerced)).toBe(false); + }); +}); + +describe("parse", () => { + it("returns the coerced value when it is valid", () => { + const validator = compile(obj({ count: num(), tags: arr(str()) }), { coerce: true }); + const result = validator.parse({ count: "42", tags: "a" }); + expect(result).toEqual({ ok: true, value: { count: 42, tags: ["a"] } }); + if (result.ok) expectTypeOf(result.value).toEqualTypeOf<{ count: number; tags: string[] }>(); + }); + + it("returns the errors of the coerced value when it is invalid", () => { + const validator = compile(obj({ count: num(), name: str() }), { coerce: true }); + expect(validator.parse({ count: "many", name: 7 })).toEqual({ + ok: false, + errors: [{ path: "/count", keyword: "type", message: "expected number, got string" }], + }); + }); +}); + +describe("pass 1: optional nulls", () => { + it("drops null in optional non-nullable properties, also nested", () => { + const schema = obj( + { + path: str(), + offset: opt(num()), + nullable: opt(nullable(str())), + metadata: obj({ enabled: opt({ type: "boolean" }) }), + }, + { open: true }, + ); + expect(coerce(schema, { path: "file.txt", offset: null, nullable: null, metadata: { enabled: null } })).toEqual({ + path: "file.txt", + nullable: null, + metadata: {}, + }); + }); + + it("keeps a null whose property is a $ref (the reference may allow it)", () => { + const schema = { + type: "object", + properties: { value: { $ref: "#/$defs/value" } }, + $defs: { value: { anyOf: [{ type: "number" }, { type: "null" }] } }, + }; + expect(coerce(schema, { value: null })).toEqual({ value: null }); + }); + + it("walks array items", () => { + const schema = { type: "array", items: obj({ a: str(), b: opt(int()) }) }; + expect(coerce(schema, [{ a: "x", b: null }])).toEqual([{ a: "x" }]); + }); +}); + +describe("pass 2: TypeBox's conversion rules on JSON Schema", () => { + it("converts primitives by type", () => { + expect(coerceValue({ type: "number" }, "42")).toBe(42); + expect(coerceValue({ type: "number" }, "")).toBe(0); + expect(coerceValue({ type: "number" }, "true")).toBe(1); + expect(coerceValue({ type: "integer" }, "3.5")).toBe(3); + expect(coerceValue({ type: "integer" }, 1.5)).toBe(1); + expect(coerceValue({ type: "boolean" }, "1")).toBe(true); + expect(coerceValue({ type: "boolean" }, "FALSE")).toBe(false); + expect(coerceValue({ type: "null" }, "null")).toBe(null); + expect(coerceValue({ type: "string" }, 42)).toBe("42"); + }); + + it("wraps a single value into an array and converts the items", () => { + expect(coerceValue({ type: "array", items: { type: "number" } }, "1")).toEqual([1]); + expect(coerceValue({ type: "array", items: { type: "string" } }, null)).toEqual(["null"]); + }); + + it("converts tuple items in place and never wraps for a tuple", () => { + const tuple = { type: "array", prefixItems: [{ type: "number" }, { type: "boolean" }], items: false }; + expect(coerceValue(tuple, ["1", "true"])).toEqual([1, true]); + expect(coerceValue(tuple, "1")).toBe("1"); + }); + + it("converts const and enum values only when the result is a member", () => { + expect(coerceValue({ const: 1 }, "1")).toBe(1); + expect(coerceValue({ const: 1 }, "2")).toBe("2"); + expect(coerceValue({ enum: [1, 2, "hello"] }, "2")).toBe(2); + expect(coerceValue({ enum: [1, 2, "hello"] }, "world")).toBe("world"); + }); + + it("keeps a value one union branch accepts, else takes the first branch whose conversion the union accepts", () => { + const union = { anyOf: [{ type: "number" }, { type: "null" }] }; + expect(coerceValue(union, null)).toBe(null); + expect(coerceValue(union, "42")).toBe(42); + expect(coerceValue(union, "null")).toBe(null); + expect(coerceValue({ oneOf: [{ type: "boolean" }, { type: "number" }] }, "1")).toBe(true); + }); + + it("converts allOf with each member in turn", () => { + const schema = { allOf: [obj({ a: num() }, { open: true }), obj({ b: { type: "boolean" } }, { open: true })] }; + expect(coerce(schema, { a: "1", b: "true" })).toEqual({ a: 1, b: true }); + }); + + it("follows $ref", () => { + const schema = { + type: "object", + properties: { item: { $ref: "#/$defs/item" } }, + $defs: { item: { type: "object", properties: { id: { type: "integer" } } } }, + }; + expect(coerce(schema, { item: { id: "7" } })).toEqual({ item: { id: 7 } }); + }); + + it("converts patternProperties and additionalProperties", () => { + const schema = { + type: "object", + properties: { a: { type: "number" } }, + patternProperties: { "^n_": { type: "number" } }, + additionalProperties: { type: "boolean" }, + }; + expect(coerce(schema, { a: "1", n_x: "2", other: "true" })).toEqual({ a: 1, n_x: 2, other: true }); + }); +}); + +describe("pass 3: primitive coercion by type", () => { + it("coerces like Ajv's coerceTypes", () => { + const cases: Array<[unknown, unknown, unknown]> = [ + [{ type: "number" }, true, 1], + [{ type: "number" }, null, 0], + [{ type: "integer" }, "42", 42], + [{ type: "boolean" }, "true", true], + [{ type: "boolean" }, 0, false], + [{ type: "string" }, true, "true"], + [{ type: "null" }, "", null], + [{ type: "null" }, 0, null], + [{ type: "null" }, false, null], + ]; + for (const [schema, input, expected] of cases) expect(coerceValue(schema, input)).toEqual(expected); + }); + + it("leaves a value that matches one of a type array's types, else takes the first coercible type", () => { + expect(coerceValue({ type: ["number", "string"] }, "1")).toBe("1"); + expect(coerceValue({ type: ["boolean", "number"] }, "1")).toBe(1); + expect(coerceValue({ type: ["integer", "null"] }, "5")).toBe(5); + }); + + it("accepts null for nullable array schemas with items", () => { + expect(coerceValue({ type: ["array", "null"], items: { type: "string" } }, null)).toBe(null); + }); + + it("coerces nullable unions when the value matches no branch", () => { + expect(coerceValue({ anyOf: [{ type: "number" }, { type: "null" }] }, "42")).toBe(42); + expect(coerceValue({ oneOf: [{ type: "number" }, { type: "null" }] }, null)).toBe(null); + }); + + it("leaves values no rule coerces", () => { + const validator = compile( + { type: "object", properties: { value: { type: "integer" } }, required: ["value"] }, + { coerce: true }, + ); + for (const input of ["abc", "4x", {}, ["1", "2"]]) { + const coerced = validator.coerce({ value: input }); + expect(validator.check(coerced)).toBe(false); + } + }); +}); + +describe("where the passes conflict, the conversion rules win", () => { + it("turns null into 'null' for a required string (pass 3 alone would give '')", () => { + expect(coerceValue({ type: "string" }, null)).toBe("null"); + }); + + it("turns '' into 0 for a number (pass 3 leaves an empty string)", () => { + expect(coerceValue({ type: "number" }, "")).toBe(0); + }); + + it("turns '1' and '0' into booleans (pass 3 takes only 'true', 'false', 1 and 0)", () => { + expect(coerceValue({ type: "boolean" }, "0")).toBe(false); + }); + + it("truncates '42.1' for an integer (pass 3 rejects it)", () => { + expect(coerceValue({ type: "integer" }, "42.1")).toBe(42); + }); + + it("drops an optional null before converting it (pass 2 alone would give 'null')", () => { + expect(coerce(obj({ name: opt(str()) }), { name: null })).toEqual({}); + }); +}); + +describe("recursive schemas", () => { + const tree = { + type: "object", + properties: { t: { $ref: "#/$defs/Tree" } }, + $defs: { Tree: { anyOf: [{ type: "string" }, { type: "array", items: { $ref: "#/$defs/Tree" } }] } }, + }; + const json = { + type: "object", + properties: { data: { $ref: "#/$defs/Json" } }, + required: ["data"], + $defs: { + Json: { + anyOf: [ + { type: "string" }, + { type: "number" }, + { type: "boolean" }, + { type: "array", items: { $ref: "#/$defs/Json" } }, + { type: "object", additionalProperties: { $ref: "#/$defs/Json" } }, + ], + }, + }, + }; + + it("convert through a recursive union by its first accepting branch", () => { + const validator = compile(tree, { coerce: true }); + expect(validator.parse({ t: ["a", ["b"]] })).toEqual({ ok: true, value: { t: ["a", ["b"]] } }); + expect(validator.parse({ t: 5 })).toEqual({ ok: true, value: { t: "5" } }); + expect(validator.parse({ t: [5] })).toEqual({ ok: true, value: { t: ["5"] } }); + expect(compile(json, { coerce: true }).parse({ data: { a: [1, null] } })).toEqual({ + ok: true, + value: { data: { a: [1, "null"] } }, + }); + }); + + it("leave a value a recursive schema cannot convert as it is, for the check to report", () => { + const validator = compile(tree, { coerce: true }); + const result = validator.parse({ t: {} }); + expect(result.ok).toBe(false); + expect(validator.coerce({ t: { a: 1 } })).toEqual({ t: { a: 1 } }); + }); + + it("convert a scalar-or-array schema that refers to itself", () => { + const schema = { + type: "object", + properties: { + value: { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#/properties/value" } }] }, + }, + required: ["value"], + }; + const validator = compile(schema, { coerce: true }); + expect(validator.parse({ value: "42" })).toEqual({ ok: true, value: { value: 42 } }); + expect(validator.parse({ value: ["1", ["2"]] })).toEqual({ ok: true, value: { value: [1, [2]] } }); + expect(validator.parse({ value: {} }).ok).toBe(false); + }); + + it("stop at a schema that cycles through shared JS objects instead of $ref", () => { + const node: { anyOf: unknown[] } = { anyOf: [{ type: "number" }] }; + node.anyOf.push({ type: "array", items: node }); + const validator = compile(node, { coerce: true }); + expect(validator.parse("42")).toEqual({ ok: true, value: 42 }); + expect(validator.parse(["1", ["2"]])).toEqual({ ok: true, value: [1, [2]] }); + expect(validator.parse({}).ok).toBe(false); + }); + + it("reset the re-entry state between calls and leave the input unchanged", () => { + const validator = compile( + { anyOf: [{ type: "number" }, { type: "array", items: { $ref: "#" } }] }, + { coerce: true }, + ); + for (let i = 0; i < 20; i++) { + expect(validator.parse({}).ok).toBe(false); + expect(validator.parse("42")).toEqual({ ok: true, value: 42 }); + const input = [1, "2", ["3"]]; + expect(validator.parse(input)).toEqual({ ok: true, value: [1, 2, [3]] }); + expect(input).toEqual([1, "2", ["3"]]); + } + }); + + it("convert and explain a deep value that every branch of a recursive union rejects, in linear time", () => { + // Each object branch converts and explains every child: without memoizing per call this is 2^depth work. + const S = { + anyOf: [ + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["a"] }, + { type: "object", additionalProperties: { $ref: "#/$defs/S" }, required: ["b"] }, + ], + }; + const validator = compile( + { type: "object", properties: { s: { $ref: "#/$defs/S" } }, $defs: { S } }, + { coerce: true }, + ); + let value: unknown = { x: 1 }; + for (let i = 0; i < 40; i++) value = { c: value }; + const result = validator.parse({ s: value }); + expect(result.ok).toBe(false); + const missing = Array.from({ length: 41 }, (_, i) => ({ + path: `/s${"/c".repeat(i)}/a`, + keyword: "required", + message: "missing", + })); + const leaf = { path: `/s${"/c".repeat(40)}/x`, keyword: "type", message: "expected object, got integer" }; + expect(result).toEqual({ ok: false, errors: [...missing, leaf] }); + }); + + it("convert a TypeBox Cyclic union", () => { + const Tree = Type.Cyclic({ Tree: Type.Union([Type.String(), Type.Array(Type.Ref("Tree"))]) }, "Tree"); + const validator = compile(Type.Object({ t: Tree }), { coerce: true }); + expect(validator.parse({ t: [5] })).toEqual({ ok: true, value: { t: ["5"] } }); + expect(validator.parse({ t: { a: 1 } }).ok).toBe(false); + }); +}); + +describe("member keywords across the passes", () => { + it("keep a pattern-matched property from additionalProperties in every pass", () => { + const record = Type.Record(Type.String(), Type.Number(), { additionalProperties: Type.String() }); + expect(compile(record, { coerce: true }).parse({ n1: "1" })).toEqual({ ok: true, value: { n1: 1 } }); + const plain = { + type: "object", + patternProperties: { "^n_": { type: ["integer", "null"] } }, + additionalProperties: { type: "string" }, + }; + expect(coerce(plain, { n_a: "5", other: 7 })).toEqual({ n_a: 5, other: "7" }); + }); + + it("drop optional nulls inside declared, pattern and additional property values alike", () => { + const item = obj({ n: opt(int({ minimum: 1 })) }); + const schemas = [ + obj({ entry: item }), + map(item), + { type: "object", patternProperties: { "^entry$": item }, additionalProperties: false }, + ]; + for (const schema of schemas) { + const input = { entry: { n: null } }; + expect(compile(schema, { coerce: true }).parse(input)).toEqual({ ok: true, value: { entry: {} } }); + expect(input).toEqual({ entry: { n: null } }); + } + }); + + it("keep a null under pattern and additional properties: their keys are not optional properties", () => { + const schema = { type: "object", additionalProperties: { type: ["integer", "null"] } }; + expect(compile(schema, { coerce: true }).parse({ n: null })).toEqual({ ok: true, value: { n: null } }); + }); + + it("drop optional nulls in tuple items as in array items", () => { + const item = obj({ n: opt(int()) }); + expect(compile(arr(item), { coerce: true }).coerce([{ n: null }])).toEqual([{}]); + expect(compile(tuple([item]), { coerce: true }).coerce([{ n: null }])).toEqual([{}]); + }); + + it("coerce tuple items by type in pass 3", () => { + const schema = { type: "array", prefixItems: [{ type: ["integer", "null"] }, { type: ["boolean", "null"] }] }; + expect(coerce(schema, ["5", "true"])).toEqual([5, true]); + }); + + it("convert the properties a union node declares besides its branches", () => { + const schema = { + type: "object", + properties: { a: { type: "integer" }, b: { type: "integer" } }, + anyOf: [{ required: ["a"] }, { required: ["b"] }], + }; + expect(compile(schema, { coerce: true }).parse({ a: "3.5" })).toEqual({ ok: true, value: { a: 3 } }); + }); + + for (const keyword of ["anyOf", "oneOf"]) { + it(`check ${keyword} candidates after the node's own properties convert`, () => { + const schema = { + type: "object", + [keyword]: [ + { type: "object", properties: { a: { type: "array", items: { type: "integer" } } }, required: ["a"] }, + { type: "object", properties: { b: { type: "boolean" } }, required: ["b"] }, + ], + properties: { c: { type: "integer" } }, + required: ["c"], + }; + const input = { a: "3", c: "2" }; + expect(compile(schema, { coerce: true }).parse(input)).toEqual({ ok: true, value: { a: [3], c: 2 } }); + expect(input).toEqual({ a: "3", c: "2" }); + }); + } +}); diff --git a/packages/schema/test/conformance.test.ts b/packages/schema/test/conformance.test.ts new file mode 100644 index 000000000..5702220c5 --- /dev/null +++ b/packages/schema/test/conformance.test.ts @@ -0,0 +1,51 @@ +// The official JSON-Schema-Test-Suite, drafts 2020-12 and 7: every test, including optional and format tests. +// A test listed in KNOWN_FAILURES must fail; when one starts passing, remove it from the list. + +import { describe, expect, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; +import { suiteFiles, suiteSchemas } from "./suite.ts"; + +const KNOWN_FAILURES = new Set([ + // Vocabularies switched off by a custom meta-schema are not supported. + "draft2020-12/vocabulary > schema that uses custom metaschema with with no validation vocabulary > no validation: invalid number, but it still validates", + "draft2020-12/optional/format-assertion > schema that uses custom metaschema with format-assertion: false > format-assertion: false: invalid string", + "draft2020-12/optional/format-assertion > schema that uses custom metaschema with format-assertion: true > format-assertion: true: invalid string", + // A $ref to a document of another draft is evaluated with the compile's draft. + "draft2020-12/optional/cross-draft > refs to historic drafts are processed as historic drafts > first item not a string is valid", + "draft7/optional/cross-draft > refs to future drafts are processed as future drafts > missing bar is invalid", + // contentMediaType and contentEncoding are annotations. + "draft7/optional/content > validation of string-encoded content based on media type > an invalid JSON document", + "draft7/optional/content > validation of binary string-encoding > an invalid base64 string (% is not a valid character)", + "draft7/optional/content > validation of binary-encoded media type documents > a validly-encoded invalid JSON document", + "draft7/optional/content > validation of binary-encoded media type documents > an invalid base64 string that is valid JSON", +]); + +const schemas = suiteSchemas(); + +for (const file of suiteFiles()) { + // Format tests assert formats; everywhere else `format` is an annotation, as draft 2020-12 specifies. + const options = { + draft: file.draft, + schemas, + formats: file.name.includes("/optional/format/") ? formats : undefined, + }; + describe(file.name, () => { + for (const group of file.groups) { + describe(group.description, () => { + for (const test of group.tests) { + const known = KNOWN_FAILURES.has(`${file.name} > ${group.description} > ${test.description}`); + (known ? it.fails : it)(test.description, () => { + const validator = compile(group.schema, options); + expect(validator.check(test.data)).toBe(test.valid); + }); + it(`${test.description} (errors agree with check)`, () => { + const validator = compile(group.schema, options); + const errors = validator.errors(test.data); + expect(errors.length === 0).toBe(validator.check(test.data)); + }); + } + }); + } + }); +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/LICENSE b/packages/schema/test/fixtures/json-schema-test-suite/LICENSE new file mode 100644 index 000000000..c28adbadd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012 Julian Berman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/schema/test/fixtures/json-schema-test-suite/README.md b/packages/schema/test/fixtures/json-schema-test-suite/README.md new file mode 100644 index 000000000..324ff0a0c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/README.md @@ -0,0 +1,9 @@ +# JSON-Schema-Test-Suite (subset) + +From https://github.com/json-schema-org/JSON-Schema-Test-Suite at commit +`3f05891cac5d51ffa068d7076d1372130e00c829`, unmodified, under its MIT license (`LICENSE`). + +Only what the drafts this package supports use: + +- `tests/draft2020-12/` and `tests/draft7/`, complete (core, `optional/`, `optional/format/`); +- `remotes/`: the documents those tests reference (served as `http://localhost:1234/...` in the tests). diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChange/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolder/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/baseUriChangeFolderInSubschema/folderInteger.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json new file mode 100644 index 000000000..0d691d964 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/dependentRequired.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2019-09/dependentRequired.json", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "dependentRequired": { + "foo": ["bar"] + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json new file mode 100644 index 000000000..b5ef3928c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2019-09/ignore-prefixItems.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2019-09/ignore-prefixItems.json", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "prefixItems": [ + {"type": "string"} + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChange/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolder/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/baseUriChangeFolderInSubschema/folderInteger.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json new file mode 100644 index 000000000..07cce1dac --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-dynamicref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/detached-dynamicref.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "foo": { + "$dynamicRef": "#detached" + }, + "detached": { + "$dynamicAnchor": "detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json new file mode 100644 index 000000000..9c2dca93c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/detached-ref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/detached-ref.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "foo": { + "$ref": "#detached" + }, + "detached": { + "$anchor": "detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json new file mode 100644 index 000000000..5d2644d3f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/different-id-ref-string.json @@ -0,0 +1,5 @@ +{ + "$id": "http://localhost:1234/draft2020-12/real-id-ref-string.json", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json new file mode 100644 index 000000000..65bc0c217 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/extendible-dynamic-ref.json @@ -0,0 +1,21 @@ +{ + "description": "extendible array", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/extendible-dynamic-ref.json", + "type": "object", + "properties": { + "elements": { + "type": "array", + "items": { + "$dynamicRef": "#elements" + } + } + }, + "required": ["elements"], + "additionalProperties": false, + "$defs": { + "elements": { + "$dynamicAnchor": "elements" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json new file mode 100644 index 000000000..43a711c9d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-false.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/format-assertion-false.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/format-assertion": false + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/format-assertion" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json new file mode 100644 index 000000000..39c6b0abf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/format-assertion-true.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft2020-12/format-assertion-true.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/format-assertion": true + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/format-assertion" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json new file mode 100644 index 000000000..1f44a6313 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/integer.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json new file mode 100644 index 000000000..6565a1ee0 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/locationIndependentIdentifier.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "refToInteger": { + "$ref": "#foo" + }, + "A": { + "$anchor": "foo", + "type": "integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json new file mode 100644 index 000000000..71be8b5da --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-no-validation.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/metaschema-no-validation.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/core": true + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/applicator" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json new file mode 100644 index 000000000..a6963e548 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/metaschema-optional-vocabulary.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/metaschema-optional-vocabulary.json", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/core": true, + "http://localhost:1234/draft/2020-12/vocab/custom": false + }, + "$dynamicAnchor": "meta", + "allOf": [ + { "$ref": "https://json-schema.org/draft/2020-12/meta/validation" }, + { "$ref": "https://json-schema.org/draft/2020-12/meta/core" } + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json new file mode 100644 index 000000000..67bc33c51 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/name-defs.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "orNull": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#" + } + ] + } + }, + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json new file mode 100644 index 000000000..3d51ded5c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested-absolute-ref-to-string.json @@ -0,0 +1,9 @@ +{ + "$defs": { + "bar": { + "$id": "http://localhost:1234/draft2020-12/the-nested-id.json", + "type": "string" + } + }, + "$ref": "http://localhost:1234/draft2020-12/the-nested-id.json" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json new file mode 100644 index 000000000..29661ff9f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/foo-ref-string.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": {"$ref": "string.json"} + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json new file mode 100644 index 000000000..6607ac534 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/nested/string.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json new file mode 100644 index 000000000..acd8293c6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/prefixItems.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft2020-12/prefixItems.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "string"} + ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json new file mode 100644 index 000000000..16d30fa3a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/ref-and-defs.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/ref-and-defs.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json new file mode 100644 index 000000000..1bb4846d7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/subSchemas.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "integer": { + "type": "integer" + }, + "refToInteger": { + "$ref": "#/$defs/integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json new file mode 100644 index 000000000..b07555fb3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/tree.json @@ -0,0 +1,17 @@ +{ + "description": "tree schema, extensible", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/tree.json", + "$dynamicAnchor": "node", + + "type": "object", + "properties": { + "data": true, + "children": { + "type": "array", + "items": { + "$dynamicRef": "#node" + } + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json new file mode 100644 index 000000000..796f37021 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft2020-12/urn-ref-string.json @@ -0,0 +1,5 @@ +{ + "$id": "urn:uuid:feebdaed-ffff-0000-2020-1200deadbeef", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json new file mode 100644 index 000000000..27f2ec80a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/detached-ref.json @@ -0,0 +1,13 @@ +{ + "$id": "http://localhost:1234/draft7/detached-ref.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": { + "foo": { + "$ref": "#detached" + }, + "detached": { + "$id": "#detached", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json new file mode 100644 index 000000000..13fd0df68 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ignore-dependentRequired.json @@ -0,0 +1,7 @@ +{ + "$id": "http://localhost:1234/draft7/ignore-dependentRequired.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "dependentRequired": { + "foo": ["bar"] + } +} \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json new file mode 100644 index 000000000..e72815cd5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/locationIndependentIdentifier.json @@ -0,0 +1,11 @@ +{ + "definitions": { + "refToInteger": { + "$ref": "#foo" + }, + "A": { + "$id": "#foo", + "type": "integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json new file mode 100644 index 000000000..fceacb809 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/name.json @@ -0,0 +1,15 @@ +{ + "definitions": { + "orNull": { + "anyOf": [ + { + "type": "null" + }, + { + "$ref": "#" + } + ] + } + }, + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json new file mode 100644 index 000000000..d5929380c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/ref-and-definitions.json @@ -0,0 +1,11 @@ +{ + "$id": "http://localhost:1234/draft7/ref-and-definitions.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json new file mode 100644 index 000000000..6e9b3de35 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/draft7/subSchemas.json @@ -0,0 +1,10 @@ +{ + "definitions": { + "integer": { + "type": "integer" + }, + "refToInteger": { + "$ref": "#/definitions/integer" + } + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json new file mode 100644 index 000000000..8b50ea308 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/integer.json @@ -0,0 +1,3 @@ +{ + "type": "integer" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json new file mode 100644 index 000000000..9cd2527ed --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/foo-ref-string.json @@ -0,0 +1,6 @@ +{ + "type": "object", + "properties": { + "foo": {"$ref": "string.json"} + } +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json new file mode 100644 index 000000000..c2d48c068 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/remotes/nested/string.json @@ -0,0 +1,3 @@ +{ + "type": "string" +} diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json new file mode 100644 index 000000000..9618575e2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/additionalProperties.json @@ -0,0 +1,219 @@ +[ + { + "description": + "additionalProperties being false does not allow other properties", + "specification": [ { "core":"10.3.2.3", "quote": "The value of \"additionalProperties\" MUST be a valid JSON Schema. Boolean \"false\" forbids everything." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}}, + "patternProperties": { "^v": {} }, + "additionalProperties": false + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : "boom"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobarbaz", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "patternProperties are not additional properties", + "data": {"foo":1, "vroom": 2}, + "valid": true + } + ] + }, + { + "description": "non-ASCII pattern with additionalProperties", + "specification": [ { "core":"10.3.2.3"} ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": {"^á": {}}, + "additionalProperties": false + }, + "tests": [ + { + "description": "matching the pattern is valid", + "data": {"ármányos": 2}, + "valid": true + }, + { + "description": "not matching the pattern is invalid", + "data": {"élmény": 2}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with schema", + "specification": [ { "core":"10.3.2.3", "quote": "The value of \"additionalProperties\" MUST be a valid JSON Schema." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}}, + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional valid property is valid", + "data": {"foo" : 1, "bar" : 2, "quux" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : 12}, + "valid": false + } + ] + }, + { + "description": "additionalProperties can exist by itself", + "specification": [ { "core":"10.3.2.3", "quote": "With no other applicator applying to object instances. This validates all the instance values irrespective of their property names" } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, + { + "description": "additionalProperties are allowed by default", + "specification": [ { "core":"10.3.2.3", "quote": "Omitting this keyword has the same assertion behavior as an empty schema." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo": {}, "bar": {}} + }, + "tests": [ + { + "description": "additional properties are allowed", + "data": {"foo": 1, "bar": 2, "quux": true}, + "valid": true + } + ] + }, + { + "description": "additionalProperties does not look in applicators", + "specification":[ { "core": "10.2", "quote": "Subschemas of applicator keywords evaluate the instance completely independently such that the results of one such subschema MUST NOT impact the results of sibling subschemas." } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {"properties": {"foo": {}}} + ], + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "properties defined in allOf are not examined", + "data": {"foo": 1, "bar": true}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with null valued instance properties", + "specification": [ { "core":"10.3.2.3" } ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "additionalProperties with propertyNames", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": { + "maxLength": 5 + }, + "additionalProperties": { + "type": "number" + } + }, + "tests": [ + { + "description": "Valid against both keywords", + "data": { "apple": 4 }, + "valid": true + }, + { + "description": "Valid against propertyNames, but not additionalProperties", + "data": { "fig": 2, "pear": "available" }, + "valid": false + } + ] + }, + { + "description": "dependentSchemas with additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo2": {}}, + "dependentSchemas": { + "foo" : {}, + "foo2": { + "properties": { + "bar": {} + } + } + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "additionalProperties doesn't consider dependentSchemas", + "data": {"foo": ""}, + "valid": false + }, + { + "description": "additionalProperties can't see bar", + "data": {"bar": ""}, + "valid": false + }, + { + "description": "additionalProperties can't see bar even when foo2 is present", + "data": {"foo2": "", "bar": ""}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json new file mode 100644 index 000000000..9e87903fe --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/allOf.json @@ -0,0 +1,312 @@ +[ + { + "description": "allOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "allOf", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "mismatch second", + "data": {"foo": "baz"}, + "valid": false + }, + { + "description": "mismatch first", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "wrong type", + "data": {"foo": "baz", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "allOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"bar": {"type": "integer"}}, + "required": ["bar"], + "allOf" : [ + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + }, + { + "properties": { + "baz": {"type": "null"} + }, + "required": ["baz"] + } + ] + }, + "tests": [ + { + "description": "valid", + "data": {"foo": "quux", "bar": 2, "baz": null}, + "valid": true + }, + { + "description": "mismatch base schema", + "data": {"foo": "quux", "baz": null}, + "valid": false + }, + { + "description": "mismatch first allOf", + "data": {"bar": 2, "baz": null}, + "valid": false + }, + { + "description": "mismatch second allOf", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "mismatch both", + "data": {"bar": 2}, + "valid": false + } + ] + }, + { + "description": "allOf simple types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {"maximum": 30}, + {"minimum": 20} + ] + }, + "tests": [ + { + "description": "valid", + "data": 25, + "valid": true + }, + { + "description": "mismatch one", + "data": 35, + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true, true] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "allOf with boolean schemas, some false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with one empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with two empty schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {}, + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with the first empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + {}, + { "type": "number" } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with the last empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "nested allOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "allOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "allOf combined with anyOf, oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ { "multipleOf": 2 } ], + "anyOf": [ { "multipleOf": 3 } ], + "oneOf": [ { "multipleOf": 5 } ] + }, + "tests": [ + { + "description": "allOf: false, anyOf: false, oneOf: false", + "data": 1, + "valid": false + }, + { + "description": "allOf: false, anyOf: false, oneOf: true", + "data": 5, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: false", + "data": 3, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: true", + "data": 15, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: false", + "data": 2, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: true", + "data": 10, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: false", + "data": 6, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: true", + "data": 30, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json new file mode 100644 index 000000000..99143fa11 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anchor.json @@ -0,0 +1,120 @@ +[ + { + "description": "Location-independent identifier", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#foo", + "$defs": { + "A": { + "$anchor": "foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with absolute URI", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/bar#foo", + "$defs": { + "A": { + "$id": "http://localhost:1234/draft2020-12/bar", + "$anchor": "foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with base URI change in subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/root", + "$ref": "http://localhost:1234/draft2020-12/nested.json#foo", + "$defs": { + "A": { + "$id": "nested.json", + "$defs": { + "B": { + "$anchor": "foo", + "type": "integer" + } + } + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "same $anchor with different base uri", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/foobar", + "$defs": { + "A": { + "$id": "child1", + "allOf": [ + { + "$id": "child2", + "$anchor": "my_anchor", + "type": "number" + }, + { + "$anchor": "my_anchor", + "type": "string" + } + ] + } + }, + "$ref": "child1#my_anchor" + }, + "tests": [ + { + "description": "$ref resolves to /$defs/A/allOf/1", + "data": "a", + "valid": true + }, + { + "description": "$ref does not resolve to /$defs/A/allOf/0", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json new file mode 100644 index 000000000..89b192dbd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/anyOf.json @@ -0,0 +1,203 @@ +[ + { + "description": "anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first anyOf valid", + "data": 1, + "valid": true + }, + { + "description": "second anyOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both anyOf valid", + "data": 3, + "valid": true + }, + { + "description": "neither anyOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "anyOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "anyOf" : [ + { + "maxLength": 2 + }, + { + "minLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one anyOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both anyOf invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [true, true] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, some true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [true, false] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf complex types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first anyOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second anyOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both anyOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "neither anyOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "anyOf with one empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is valid", + "data": 123, + "valid": true + } + ] + }, + { + "description": "nested anyOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "anyOf": [ + { + "anyOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json new file mode 100644 index 000000000..6d40f23f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/boolean_schema.json @@ -0,0 +1,104 @@ +[ + { + "description": "boolean schema 'true'", + "schema": true, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "boolean schema 'false'", + "schema": false, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json new file mode 100644 index 000000000..72ea3f042 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/const.json @@ -0,0 +1,431 @@ +[ + { + "description": "const validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 2 + }, + "tests": [ + { + "description": "same value is valid", + "data": 2, + "valid": true + }, + { + "description": "another value is invalid", + "data": 5, + "valid": false + }, + { + "description": "another type is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "const with object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"foo": "bar", "baz": "bax"} + }, + "tests": [ + { + "description": "same object is valid", + "data": {"foo": "bar", "baz": "bax"}, + "valid": true + }, + { + "description": "same object with different property order is valid", + "data": {"baz": "bax", "foo": "bar"}, + "valid": true + }, + { + "description": "another object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "another type is invalid", + "data": [1, 2], + "valid": false + } + ] + }, + { + "description": "const with array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [{ "foo": "bar" }] + }, + "tests": [ + { + "description": "same array is valid", + "data": [{"foo": "bar"}], + "valid": true + }, + { + "description": "another array item is invalid", + "data": [2], + "valid": false + }, + { + "description": "array with additional items is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + }, + { + "description": "const with null", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": null + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "not null is invalid", + "data": 0, + "valid": false + } + ] + }, + { + "description": "const with false does not match 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": false + }, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "const with true does not match 1", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": true + }, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "const with [false] does not match [0]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [false] + }, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "const with [true] does not match [1]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": [true] + }, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "const with {\"a\": false} does not match {\"a\": 0}", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"a": false} + }, + "tests": [ + { + "description": "{\"a\": false} is valid", + "data": {"a": false}, + "valid": true + }, + { + "description": "{\"a\": 0} is invalid", + "data": {"a": 0}, + "valid": false + }, + { + "description": "{\"a\": 0.0} is invalid", + "data": {"a": 0.0}, + "valid": false + } + ] + }, + { + "description": "const with {\"a\": true} does not match {\"a\": 1}", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": {"a": true} + }, + "tests": [ + { + "description": "{\"a\": true} is valid", + "data": {"a": true}, + "valid": true + }, + { + "description": "{\"a\": 1} is invalid", + "data": {"a": 1}, + "valid": false + }, + { + "description": "{\"a\": 1.0} is invalid", + "data": {"a": 1.0}, + "valid": false + } + ] + }, + { + "description": "const with 0 does not match other zero-like types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 0 + }, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + } + ] + }, + { + "description": "const with 1 does not match true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 1 + }, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "const with -2.0 matches integer and float types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": -2.0 + }, + "tests": [ + { + "description": "integer -2 is valid", + "data": -2, + "valid": true + }, + { + "description": "integer 2 is invalid", + "data": 2, + "valid": false + }, + { + "description": "float -2.0 is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float 2.0 is invalid", + "data": 2.0, + "valid": false + }, + { + "description": "float -2.00001 is invalid", + "data": -2.00001, + "valid": false + } + ] + }, + { + "description": "float and integers are equal up to 64-bit representation limits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": 9007199254740992 + }, + "tests": [ + { + "description": "integer is valid", + "data": 9007199254740992, + "valid": true + }, + { + "description": "integer minus one is invalid", + "data": 9007199254740991, + "valid": false + }, + { + "description": "float is valid", + "data": 9007199254740992.0, + "valid": true + }, + { + "description": "float minus one is invalid", + "data": 9007199254740991.0, + "valid": false + } + ] + }, + { + "description": "nul characters in strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "hello\u0000there" + }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation but different codepoint", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "μ", + "$comment": "U+03BC" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "μ", + "comment": "U+03BC", + "valid": true + }, + { + "description": "character looks the same but uses a different codepoint", + "data": "µ", + "comment": "U+00B5", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "const": "ä", + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json new file mode 100644 index 000000000..08a00a753 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/contains.json @@ -0,0 +1,176 @@ +[ + { + "description": "contains keyword validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"minimum": 5} + }, + "tests": [ + { + "description": "array with item matching schema (5) is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with item matching schema (6) is valid", + "data": [3, 4, 6], + "valid": true + }, + { + "description": "array with two items matching schema (5, 6) is valid", + "data": [3, 4, 5, 6], + "valid": true + }, + { + "description": "array without items matching schema is invalid", + "data": [2, 3, 4], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "not array is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "contains keyword with const keyword", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { "const": 5 } + }, + "tests": [ + { + "description": "array with item 5 is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with two items 5 is valid", + "data": [3, 4, 5, 5], + "valid": true + }, + { + "description": "array without item 5 is invalid", + "data": [1, 2, 3, 4], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": true + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": false + }, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "non-arrays are valid", + "data": "contains does not apply to strings", + "valid": true + } + ] + }, + { + "description": "items + contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { "multipleOf": 2 }, + "contains": { "multipleOf": 3 } + }, + "tests": [ + { + "description": "matches items, does not match contains", + "data": [ 2, 4, 8 ], + "valid": false + }, + { + "description": "does not match items, matches contains", + "data": [ 3, 6, 9 ], + "valid": false + }, + { + "description": "matches both items and contains", + "data": [ 6, 12 ], + "valid": true + }, + { + "description": "matches neither items nor contains", + "data": [ 1, 5 ], + "valid": false + } + ] + }, + { + "description": "contains with false if subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { + "if": false, + "else": true + } + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null items", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json new file mode 100644 index 000000000..698f78057 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/content.json @@ -0,0 +1,131 @@ +[ + { + "description": "validation of string-encoded content based on media type", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json" + }, + "tests": [ + { + "description": "a valid JSON document", + "data": "{\"foo\": \"bar\"}", + "valid": true + }, + { + "description": "an invalid JSON document; validates true", + "data": "{:}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary string-encoding", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64 string", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "an invalid base64 string (% is not a valid character); validates true", + "data": "eyJmb28iOi%iYmFyIn0K", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document; validates true", + "data": "ezp9Cg==", + "valid": true + }, + { + "description": "an invalid base64 string that is valid JSON; validates true", + "data": "{}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents with schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contentMediaType": "application/json", + "contentEncoding": "base64", + "contentSchema": { "type": "object", "required": ["foo"], "properties": { "foo": { "type": "string" } } } + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "another valid base64-encoded JSON document", + "data": "eyJib28iOiAyMCwgImZvbyI6ICJiYXoifQ==", + "valid": true + }, + { + "description": "an invalid base64-encoded JSON document; validates true", + "data": "eyJib28iOiAyMH0=", + "valid": true + }, + { + "description": "an empty object as a base64-encoded JSON document; validates true", + "data": "e30=", + "valid": true + }, + { + "description": "an empty array as a base64-encoded JSON document", + "data": "W10=", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document; validates true", + "data": "ezp9Cg==", + "valid": true + }, + { + "description": "an invalid base64 string that is valid JSON; validates true", + "data": "{}", + "valid": true + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json new file mode 100644 index 000000000..ceb3ae271 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/default.json @@ -0,0 +1,82 @@ +[ + { + "description": "invalid type for default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "type": "integer", + "default": [] + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"foo": 13}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "invalid string value for default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "bar": { + "type": "string", + "minLength": 4, + "default": "bad" + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"bar": "good"}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "the default keyword does not do anything if the property is missing", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "alpha": { + "type": "number", + "maximum": 3, + "default": 5 + } + } + }, + "tests": [ + { + "description": "an explicit property value is checked against maximum (passing)", + "data": { "alpha": 1 }, + "valid": true + }, + { + "description": "an explicit property value is checked against maximum (failing)", + "data": { "alpha": 5 }, + "valid": false + }, + { + "description": "missing properties are not filled in with the default", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json new file mode 100644 index 000000000..da2a503bf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/defs.json @@ -0,0 +1,21 @@ +[ + { + "description": "validate definition against metaschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "https://json-schema.org/draft/2020-12/schema" + }, + "tests": [ + { + "description": "valid definition schema", + "data": {"$defs": {"foo": {"type": "integer"}}}, + "valid": true + }, + { + "description": "invalid definition schema", + "data": {"$defs": {"foo": {"type": 1}}}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json new file mode 100644 index 000000000..2baa38e9f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentRequired.json @@ -0,0 +1,152 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "empty dependents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependents required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentRequired": { + "foo\nbar": ["foo\rbar"], + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "CRLF", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "quoted quotes", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "CRLF missing dependent", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "quoted quotes missing dependent", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json new file mode 100644 index 000000000..1c5f0574a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dependentSchemas.json @@ -0,0 +1,171 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "boolean subschemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependentSchemas": { + "foo\tbar": {"minProperties": 4}, + "foo'bar": {"required": ["foo\"bar"]} + } + }, + "tests": [ + { + "description": "quoted tab", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "quoted quote", + "data": { + "foo'bar": {"foo\"bar": 1} + }, + "valid": false + }, + { + "description": "quoted tab invalid under dependent schema", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "quoted quote invalid under dependent schema", + "data": {"foo'bar": 1}, + "valid": false + } + ] + }, + { + "description": "dependent subschema incompatible with root", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + }, + "dependentSchemas": { + "foo": { + "properties": { + "bar": {} + }, + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "matches root", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "matches dependency", + "data": {"bar": 1}, + "valid": true + }, + { + "description": "matches both", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "no dependency", + "data": {"baz": 1}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json new file mode 100644 index 000000000..eba4b441b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/dynamicRef.json @@ -0,0 +1,874 @@ +[ + { + "description": "A $dynamicRef to a $dynamicAnchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-dynamicAnchor-same-schema/root", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef to an $anchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-anchor-same-schema/root", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "foo": { + "$anchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $ref to a $dynamicAnchor in the same schema resource behaves like a normal $ref to an $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/ref-dynamicAnchor-same-schema/root", + "type": "array", + "items": { "$ref": "#items" }, + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef resolves to the first $dynamicAnchor still in scope that is encountered when the schema is evaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/typical-dynamic-resolution/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "A $dynamicRef without anchor in fragment behaves identical to $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamicRef-without-anchor/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#/$defs/items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items", + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is invalid", + "data": ["foo", "bar"], + "valid": false + }, + { + "description": "An array of numbers is valid", + "data": [24, 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef with intermediate scopes that don't include a matching $dynamicAnchor does not affect dynamic scope resolution", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-with-intermediate-scopes/root", + "$ref": "intermediate-scope", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "intermediate-scope": { + "$id": "intermediate-scope", + "$ref": "list" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "An array of strings is valid", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "An array containing non-strings is invalid", + "data": ["foo", 42], + "valid": false + } + ] + }, + { + "description": "An $anchor with the same name as a $dynamicAnchor is not used for dynamic scope resolution", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-ignores-anchors/root", + "$ref": "list", + "$defs": { + "foo": { + "$anchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to satisfy the bookending requirement", + "$dynamicAnchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef without a matching $dynamicAnchor in the same schema resource behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-resolution-without-bookend/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to give the reference somewhere to resolve to when it behaves like $ref", + "$anchor": "items" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef with a non-matching $dynamicAnchor in the same schema resource behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/unmatched-dynamic-anchor/root", + "$ref": "list", + "$defs": { + "foo": { + "$dynamicAnchor": "items", + "type": "string" + }, + "list": { + "$id": "list", + "type": "array", + "items": { "$dynamicRef": "#items" }, + "$defs": { + "items": { + "$comment": "This is only needed to give the reference somewhere to resolve to when it behaves like $ref", + "$anchor": "items", + "$dynamicAnchor": "foo" + } + } + } + } + }, + "tests": [ + { + "description": "Any array is valid", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "A $dynamicRef that initially resolves to a schema with a matching $dynamicAnchor resolves to the first $dynamicAnchor in the dynamic scope", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/relative-dynamic-reference/root", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "foo": { "const": "pass" } + }, + "$ref": "extended", + "$defs": { + "extended": { + "$id": "extended", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "bar": { "$ref": "bar" } + } + }, + "bar": { + "$id": "bar", + "type": "object", + "properties": { + "baz": { "$dynamicRef": "extended#meta" } + } + } + } + }, + "tests": [ + { + "description": "The recursive part is valid against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "pass" } + } + }, + "valid": true + }, + { + "description": "The recursive part is not valid against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "fail" } + } + }, + "valid": false + } + ] + }, + { + "description": "A $dynamicRef that initially resolves to a schema without a matching $dynamicAnchor behaves like a normal $ref to $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/relative-dynamic-reference-without-bookend/root", + "$dynamicAnchor": "meta", + "type": "object", + "properties": { + "foo": { "const": "pass" } + }, + "$ref": "extended", + "$defs": { + "extended": { + "$id": "extended", + "$anchor": "meta", + "type": "object", + "properties": { + "bar": { "$ref": "bar" } + } + }, + "bar": { + "$id": "bar", + "type": "object", + "properties": { + "baz": { "$dynamicRef": "extended#meta" } + } + } + } + }, + "tests": [ + { + "description": "The recursive part doesn't need to validate against the root", + "data": { + "foo": "pass", + "bar": { + "baz": { "foo": "fail" } + } + }, + "valid": true + } + ] + }, + { + "description": "multiple dynamic paths to the $dynamicRef keyword", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-with-multiple-paths/main", + "if": { + "properties": { + "kindOfList": { "const": "numbers" } + }, + "required": ["kindOfList"] + }, + "then": { "$ref": "numberList" }, + "else": { "$ref": "stringList" }, + + "$defs": { + "genericList": { + "$id": "genericList", + "properties": { + "list": { + "items": { "$dynamicRef": "#itemType" } + } + }, + "$defs": { + "defaultItemType": { + "$comment": "Only needed to satisfy bookending requirement", + "$dynamicAnchor": "itemType" + } + } + }, + "numberList": { + "$id": "numberList", + "$defs": { + "itemType": { + "$dynamicAnchor": "itemType", + "type": "number" + } + }, + "$ref": "genericList" + }, + "stringList": { + "$id": "stringList", + "$defs": { + "itemType": { + "$dynamicAnchor": "itemType", + "type": "string" + } + }, + "$ref": "genericList" + } + } + }, + "tests": [ + { + "description": "number list with number values", + "data": { + "kindOfList": "numbers", + "list": [1.1] + }, + "valid": true + }, + { + "description": "number list with string values", + "data": { + "kindOfList": "numbers", + "list": ["foo"] + }, + "valid": false + }, + { + "description": "string list with number values", + "data": { + "kindOfList": "strings", + "list": [1.1] + }, + "valid": false + }, + { + "description": "string list with string values", + "data": { + "kindOfList": "strings", + "list": ["foo"] + }, + "valid": true + } + ] + }, + { + "description": "after leaving a dynamic scope, it is not used by a $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-leaving-dynamic-scope/main", + "if": { + "$id": "first_scope", + "$defs": { + "thingy": { + "$comment": "this is first_scope#thingy", + "$dynamicAnchor": "thingy", + "type": "number" + } + } + }, + "then": { + "$id": "second_scope", + "$ref": "start", + "$defs": { + "thingy": { + "$comment": "this is second_scope#thingy, the final destination of the $dynamicRef", + "$dynamicAnchor": "thingy", + "type": "null" + } + } + }, + "$defs": { + "start": { + "$comment": "this is the landing spot from $ref", + "$id": "start", + "$dynamicRef": "inner_scope#thingy" + }, + "thingy": { + "$comment": "this is the first stop for the $dynamicRef", + "$id": "inner_scope", + "$dynamicAnchor": "thingy", + "type": "string" + } + } + }, + "tests": [ + { + "description": "string matches /$defs/thingy, but the $dynamicRef does not stop here", + "data": "a string", + "valid": false + }, + { + "description": "first_scope is not in dynamic scope for the $dynamicRef", + "data": 42, + "valid": false + }, + { + "description": "/then/$defs/thingy is the final stop for the $dynamicRef", + "data": null, + "valid": true + } + ] + }, + { + "description": "strict-tree schema, guards against misspelled properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-tree.json", + "$dynamicAnchor": "node", + + "$ref": "tree.json", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "instance with misspelled field", + "data": { + "children": [{ + "daat": 1 + }] + }, + "valid": false + }, + { + "description": "instance with correct field", + "data": { + "children": [{ + "data": 1 + }] + }, + "valid": true + } + ] + }, + { + "description": "tests for implementation dynamic anchor and reference link", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible.json", + "$ref": "extendible-dynamic-ref.json", + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref and $dynamicAnchor are independent of order - $defs first", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible-allof-defs-first.json", + "allOf": [ + { + "$ref": "extendible-dynamic-ref.json" + }, + { + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + } + ] + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref and $dynamicAnchor are independent of order - $ref first", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/strict-extendible-allof-ref-first.json", + "allOf": [ + { + "$defs": { + "elements": { + "$dynamicAnchor": "elements", + "properties": { + "a": true + }, + "required": ["a"], + "additionalProperties": false + } + } + }, + { + "$ref": "extendible-dynamic-ref.json" + } + ] + }, + "tests": [ + { + "description": "incorrect parent schema", + "data": { + "a": true + }, + "valid": false + }, + { + "description": "incorrect extended schema", + "data": { + "elements": [ + { "b": 1 } + ] + }, + "valid": false + }, + { + "description": "correct extended schema", + "data": { + "elements": [ + { "a": 1 } + ] + }, + "valid": true + } + ] + }, + { + "description": "$ref to $dynamicRef finds detached $dynamicAnchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/detached-dynamicref.json#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$dynamicRef points to a boolean schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "true": true, + "false": false + }, + "properties": { + "true": { + "$dynamicRef": "#/$defs/true" + }, + "false": { + "$dynamicRef": "#/$defs/false" + } + } + }, + "tests": [ + { + "description": "follow $dynamicRef to a true schema", + "data": { "true": 1 }, + "valid": true + }, + { + "description": "follow $dynamicRef to a false schema", + "data": { "false": 1 }, + "valid": false + } + ] + }, + { + "description": "$dynamicRef skips over intermediate resources - direct reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-skips-intermediate-resource/main", + "type": "object", + "properties": { + "bar-item": { + "$ref": "item" + } + }, + "$defs": { + "bar": { + "$id": "bar", + "type": "array", + "items": { + "$ref": "item" + }, + "$defs": { + "item": { + "$id": "item", + "type": "object", + "properties": { + "content": { + "$dynamicRef": "#content" + } + }, + "$defs": { + "defaultContent": { + "$dynamicAnchor": "content", + "type": "integer" + } + } + }, + "content": { + "$dynamicAnchor": "content", + "type": "string" + } + } + } + } + }, + "tests": [ + { + "description": "integer property passes", + "data": { "bar-item": { "content": 42 } }, + "valid": true + }, + { + "description": "string property fails", + "data": { "bar-item": { "content": "value" } }, + "valid": false + } + ] + }, + { + "description": "$dynamicRef avoids the root of each schema, but scopes are still registered", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-avoids-root-of-each-schema/base", + "$ref": "first#/$defs/stuff", + "$defs": { + "first": { + "$id": "first", + "$defs": { + "stuff": { + "$ref": "second#/$defs/stuff" + }, + "length": { + "$comment": "unused, because there is no $dynamicAnchor here", + "maxLength": 1 + } + } + }, + "second": { + "$id": "second", + "$defs": { + "stuff": { + "$ref": "third#/$defs/stuff" + }, + "length": { + "$dynamicAnchor": "length", + "maxLength": 2 + } + } + }, + "third": { + "$id": "third", + "$defs": { + "stuff": { + "$dynamicRef": "#length" + }, + "length": { + "$dynamicAnchor": "length", + "maxLength": 3 + } + } + } + } + }, + "tests": [ + { + "description": "data is sufficient for schema at second#/$defs/length", + "data": "hi", + "valid": true + }, + { + "description": "data is not sufficient for schema at second#/$defs/length", + "data": "hey", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json new file mode 100644 index 000000000..dbc5e0d31 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/enum.json @@ -0,0 +1,419 @@ +[ + { + "description": "simple enum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [1, 2, 3] + }, + "tests": [ + { + "description": "one of the enum is valid", + "data": 1, + "valid": true + }, + { + "description": "something else is invalid", + "data": 4, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [6, "foo", [], true, {"foo": 12}] + }, + "tests": [ + { + "description": "one of the enum is valid", + "data": [], + "valid": true + }, + { + "description": "something else is invalid", + "data": null, + "valid": false + }, + { + "description": "objects are deep compared", + "data": {"foo": false}, + "valid": false + }, + { + "description": "valid object matches", + "data": {"foo": 12}, + "valid": true + }, + { + "description": "extra properties in object is invalid", + "data": {"foo": 12, "boo": 42}, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum-with-null validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [6, null] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is valid", + "data": 6, + "valid": true + }, + { + "description": "something else is invalid", + "data": "test", + "valid": false + } + ] + }, + { + "description": "enums in properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type":"object", + "properties": { + "foo": {"enum":["foo"]}, + "bar": {"enum":["bar"]} + }, + "required": ["bar"] + }, + "tests": [ + { + "description": "both properties are valid", + "data": {"foo":"foo", "bar":"bar"}, + "valid": true + }, + { + "description": "wrong foo value", + "data": {"foo":"foot", "bar":"bar"}, + "valid": false + }, + { + "description": "wrong bar value", + "data": {"foo":"foo", "bar":"bart"}, + "valid": false + }, + { + "description": "missing optional property is valid", + "data": {"bar":"bar"}, + "valid": true + }, + { + "description": "missing required property is invalid", + "data": {"foo":"foo"}, + "valid": false + }, + { + "description": "missing all properties is invalid", + "data": {}, + "valid": false + } + ] + }, + { + "description": "enum with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": ["foo\nbar", "foo\rbar"] + }, + "tests": [ + { + "description": "member 1 is valid", + "data": "foo\nbar", + "valid": true + }, + { + "description": "member 2 is valid", + "data": "foo\rbar", + "valid": true + }, + { + "description": "another string is invalid", + "data": "abc", + "valid": false + } + ] + }, + { + "description": "enum with false does not match 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [false] + }, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "enum with [false] does not match [0]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[false]] + }, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "enum with true does not match 1", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [true] + }, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "enum with [true] does not match [1]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[true]] + }, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "enum with 0 does not match false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [0] + }, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + } + ] + }, + { + "description": "enum with [0] does not match [false]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[0]] + }, + "tests": [ + { + "description": "[false] is invalid", + "data": [false], + "valid": false + }, + { + "description": "[0] is valid", + "data": [0], + "valid": true + }, + { + "description": "[0.0] is valid", + "data": [0.0], + "valid": true + } + ] + }, + { + "description": "enum with 1 does not match true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [1] + }, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "enum with [1] does not match [true]", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [[1]] + }, + "tests": [ + { + "description": "[true] is invalid", + "data": [true], + "valid": false + }, + { + "description": "[1] is valid", + "data": [1], + "valid": true + }, + { + "description": "[1.0] is valid", + "data": [1.0], + "valid": true + } + ] + }, + { + "description": "nul characters in strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [ "hello\u0000there" ] + }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "empty enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": [] + }, + "tests": [ + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "number is invalid", + "data": 42, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": [], + "valid": false + }, + { + "description": "boolean is invalid", + "data": false, + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "enum": ["ä"], + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json new file mode 100644 index 000000000..05db23351 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMaximum.json @@ -0,0 +1,31 @@ +[ + { + "description": "exclusiveMaximum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMaximum": 3.0 + }, + "tests": [ + { + "description": "below the exclusiveMaximum is valid", + "data": 2.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 3.0, + "valid": false + }, + { + "description": "above the exclusiveMaximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json new file mode 100644 index 000000000..00af9d7ff --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/exclusiveMinimum.json @@ -0,0 +1,31 @@ +[ + { + "description": "exclusiveMinimum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMinimum": 1.1 + }, + "tests": [ + { + "description": "above the exclusiveMinimum is valid", + "data": 1.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "below the exclusiveMinimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json new file mode 100644 index 000000000..01adcbda3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/format.json @@ -0,0 +1,838 @@ +[ + { + "description": "email format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid email string is only an annotation by default", + "data": "2962", + "valid": true + } + ] + }, + { + "description": "idn-email format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid idn-email string is only an annotation by default", + "data": "2962", + "valid": true + } + ] + }, + { + "description": "regex format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid regex string is only an annotation by default", + "data": "^(abc]", + "valid": true + } + ] + }, + { + "description": "ipv4 format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv4" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid ipv4 string is only an annotation by default", + "data": "127.0.0.0.1", + "valid": true + } + ] + }, + { + "description": "ipv6 format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv6" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid ipv6 string is only an annotation by default", + "data": "12345::", + "valid": true + } + ] + }, + { + "description": "idn-hostname format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid idn-hostname string is only an annotation by default", + "data": "〮실례.테스트", + "valid": true + } + ] + }, + { + "description": "hostname format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid hostname string is only an annotation by default", + "data": "-a-host-name-that-starts-with--", + "valid": true + } + ] + }, + { + "description": "date format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid date string is only an annotation by default", + "data": "06/19/1963", + "valid": true + } + ] + }, + { + "description": "date-time format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date-time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid date-time string is only an annotation by default", + "data": "1990-02-31T15:59:60.123-08:00", + "valid": true + } + ] + }, + { + "description": "time format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid time string is only an annotation by default", + "data": "08:30:06 PST", + "valid": true + } + ] + }, + { + "description": "json-pointer format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid json-pointer string is only an annotation by default", + "data": "/foo/bar~", + "valid": true + } + ] + }, + { + "description": "relative-json-pointer format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "relative-json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid relative-json-pointer string is only an annotation by default", + "data": "/foo/bar", + "valid": true + } + ] + }, + { + "description": "iri format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid iri string is only an annotation by default", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": true + } + ] + }, + { + "description": "iri-reference format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid iri-reference string is only an annotation by default", + "data": "\\\\WINDOWS\\filëßåré", + "valid": true + } + ] + }, + { + "description": "uri format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri string is only an annotation by default", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + } + ] + }, + { + "description": "uri-reference format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri-reference string is only an annotation by default", + "data": "\\\\WINDOWS\\fileshare", + "valid": true + } + ] + }, + { + "description": "uri-template format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-template" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uri-template string is only an annotation by default", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": true + } + ] + }, + { + "description": "uuid format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uuid" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid uuid string is only an annotation by default", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1638", + "valid": true + } + ] + }, + { + "description": "duration format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "duration" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "invalid duration string is only an annotation by default", + "data": "PT1D", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json new file mode 100644 index 000000000..9abf26f71 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/if-then-else.json @@ -0,0 +1,308 @@ +[ + { + "description": "ignore if without then or else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone if", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone if", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore then without if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "then": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone then", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone then", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore else without if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "else": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone else", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone else", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "if and then without else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid when if test fails", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if and else without then", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid when if test passes", + "data": -1, + "valid": true + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "validate against correct branch, then vs else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "non-interference across combined schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "if": { + "exclusiveMaximum": 0 + } + }, + { + "then": { + "minimum": -10 + } + }, + { + "else": { + "multipleOf": 2 + } + } + ] + }, + "tests": [ + { + "description": "valid, but would have been invalid through then", + "data": -100, + "valid": true + }, + { + "description": "valid, but would have been invalid through else", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": true, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema true in if always chooses the then path (valid)", + "data": "then", + "valid": true + }, + { + "description": "boolean schema true in if always chooses the then path (invalid)", + "data": "else", + "valid": false + } + ] + }, + { + "description": "if with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": false, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema false in if always chooses the else path (invalid)", + "data": "then", + "valid": false + }, + { + "description": "boolean schema false in if always chooses the else path (valid)", + "data": "else", + "valid": true + } + ] + }, + { + "description": "if appears at the end when serialized (keyword processing sequence)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "then": { "const": "yes" }, + "else": { "const": "other" }, + "if": { "maxLength": 4 } + }, + "tests": [ + { + "description": "yes redirects to then and passes", + "data": "yes", + "valid": true + }, + { + "description": "other redirects to else and passes", + "data": "other", + "valid": true + }, + { + "description": "no redirects to then and fails", + "data": "no", + "valid": false + }, + { + "description": "invalid redirects to else and fails", + "data": "invalid", + "valid": false + } + ] + }, + { + "description": "then: false fails when condition matches", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { "const": 1 }, + "then": false + }, + "tests": [ + { + "description": "matches if → then=false → invalid", + "data": 1, + "valid": false + }, + { + "description": "does not match if → then ignored → valid", + "data": 2, + "valid": true + } + ] + }, + { + "description": "else: false fails when condition does not match", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { "const": 1 }, + "else": false + }, + "tests": [ + { + "description": "matches if → else ignored → valid", + "data": 1, + "valid": true + }, + { + "description": "does not match if → else executes → invalid", + "data": 2, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json new file mode 100644 index 000000000..46f157a35 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/infinite-loop-detection.json @@ -0,0 +1,37 @@ +[ + { + "description": "evaluating the same schema location against the same data location twice is not a sign of an infinite loop", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "int": { "type": "integer" } + }, + "allOf": [ + { + "properties": { + "foo": { + "$ref": "#/$defs/int" + } + } + }, + { + "additionalProperties": { + "$ref": "#/$defs/int" + } + } + ] + }, + "tests": [ + { + "description": "passing case", + "data": { "foo": 1 }, + "valid": true + }, + { + "description": "failing case", + "data": { "foo": "a string" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json new file mode 100644 index 000000000..6a3e1cf26 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/items.json @@ -0,0 +1,304 @@ +[ + { + "description": "a schema given for items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": {"type": "integer"} + }, + "tests": [ + { + "description": "valid items", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "wrong type of items", + "data": [1, "x"], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "length": 1 + }, + "valid": true + } + ] + }, + { + "description": "items with boolean schema (true)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": true + }, + "tests": [ + { + "description": "any array is valid", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schema (false)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": false + }, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": [ 1, "foo", true ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items and subitems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "item": { + "type": "array", + "items": false, + "prefixItems": [ + { "$ref": "#/$defs/sub-item" }, + { "$ref": "#/$defs/sub-item" } + ] + }, + "sub-item": { + "type": "object", + "required": ["foo"] + } + }, + "type": "array", + "items": false, + "prefixItems": [ + { "$ref": "#/$defs/item" }, + { "$ref": "#/$defs/item" }, + { "$ref": "#/$defs/item" } + ] + }, + "tests": [ + { + "description": "valid items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": true + }, + { + "description": "too many items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "too many sub-items", + "data": [ + [ {"foo": null}, {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong item", + "data": [ + {"foo": null}, + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong sub-item", + "data": [ + [ {}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "fewer items is valid", + "data": [ + [ {"foo": null} ], + [ {"foo": null} ] + ], + "valid": true + } + ] + }, + { + "description": "nested items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "valid nested array", + "data": [[[[1]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": true + }, + { + "description": "nested array with invalid type", + "data": [[[["1"]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": false + }, + { + "description": "not deep enough", + "data": [[[1], [2],[3]], [[4], [5], [6]]], + "valid": false + } + ] + }, + { + "description": "prefixItems with no additional items allowed", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{}, {}, {}], + "items": false + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "fewer number of items present (1)", + "data": [ 1 ], + "valid": true + }, + { + "description": "fewer number of items present (2)", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "equal number of items present", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "additional items are not permitted", + "data": [ 1, 2, 3, 4 ], + "valid": false + } + ] + }, + { + "description": "items does not look in applicators, valid case", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "prefixItems": [ { "minimum": 3 } ] } + ], + "items": { "minimum": 5 } + }, + "tests": [ + { + "description": "prefixItems in allOf does not constrain items, invalid case", + "data": [ 3, 5 ], + "valid": false + }, + { + "description": "prefixItems in allOf does not constrain items, valid case", + "data": [ 5, 5 ], + "valid": true + } + ] + }, + { + "description": "prefixItems validation adjusts the starting index for items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ { "type": "string" } ], + "items": { "type": "integer" } + }, + "tests": [ + { + "description": "valid items", + "data": [ "x", 2, 3 ], + "valid": true + }, + { + "description": "wrong type of second item", + "data": [ "x", "y" ], + "valid": false + } + ] + }, + { + "description": "items with heterogeneous array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{}], + "items": false + }, + "tests": [ + { + "description": "heterogeneous invalid instance", + "data": [ "foo", "bar", 37 ], + "valid": false + }, + { + "description": "valid instance", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "items with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json new file mode 100644 index 000000000..9a42b45d6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxContains.json @@ -0,0 +1,123 @@ +[ + { + "description": "maxContains without contains is ignored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxContains": 1 + }, + "tests": [ + { + "description": "one item valid against lone maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "two items still valid against lone maxContains", + "data": [ 1, 2 ], + "valid": true + } + ] + }, + { + "description": "maxContains with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, valid maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "all elements match, invalid maxContains", + "data": [ 1, 1 ], + "valid": false + }, + { + "description": "some elements match, valid maxContains", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "some elements match, invalid maxContains", + "data": [ 1, 2, 1 ], + "valid": false + } + ] + }, + { + "description": "maxContains with contains, value with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1.0 + }, + "tests": [ + { + "description": "one element matches, valid maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "too many elements match, invalid maxContains", + "data": [ 1, 1 ], + "valid": false + } + ] + }, + { + "description": "minContains < maxContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 1, + "maxContains": 3 + }, + "tests": [ + { + "description": "actual < minContains < maxContains", + "data": [ ], + "valid": false + }, + { + "description": "minContains < actual < maxContains", + "data": [ 1, 1 ], + "valid": true + }, + { + "description": "minContains < maxContains < actual", + "data": [ 1, 1, 1, 1 ], + "valid": false + } + ] + }, + { + "description": "maxContains = 0 with minContains = 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0, + "maxContains": 0 + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "one matching item", + "data": [ 1 ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json new file mode 100644 index 000000000..f6a6b7c9a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxItems.json @@ -0,0 +1,50 @@ +[ + { + "description": "maxItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxItems": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "foobar", + "valid": true + } + ] + }, + { + "description": "maxItems validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxItems": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json new file mode 100644 index 000000000..7462726d7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxLength.json @@ -0,0 +1,55 @@ +[ + { + "description": "maxLength validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxLength": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + }, + { + "description": "two graphemes is long enough", + "data": "\uD83D\uDCA9\uD83D\uDCA9", + "valid": true + } + ] + }, + { + "description": "maxLength validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxLength": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json new file mode 100644 index 000000000..9cb745cad --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maxProperties.json @@ -0,0 +1,94 @@ +[ + { + "description": "maxProperties validation", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 2 + }, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "maxProperties validation with a decimal", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 2.0 + }, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + } + ] + }, + { + "description": "maxProperties = 0 means the object is empty", + "specification": [ + { + "validation": "6.5.1" + } + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maxProperties": 0 + }, + "tests": [ + { + "description": "no properties is valid", + "data": {}, + "valid": true + }, + { + "description": "one property is invalid", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json new file mode 100644 index 000000000..7eeab2877 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/maximum.json @@ -0,0 +1,60 @@ +[ + { + "description": "maximum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 3.0 + }, + "tests": [ + { + "description": "below the maximum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 3.0, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "maximum validation with unsigned integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 300 + }, + "tests": [ + { + "description": "below the maximum is valid", + "data": 299.97, + "valid": true + }, + { + "description": "boundary point integer is valid", + "data": 300, + "valid": true + }, + { + "description": "boundary point float is valid", + "data": 300.00, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 300.5, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json new file mode 100644 index 000000000..ee72d7d62 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minContains.json @@ -0,0 +1,224 @@ +[ + { + "description": "minContains without contains is ignored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minContains": 1 + }, + "tests": [ + { + "description": "one item valid against lone minContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "zero items still valid against lone minContains", + "data": [], + "valid": true + } + ] + }, + { + "description": "minContains=1 with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "no elements match", + "data": [ 2 ], + "valid": false + }, + { + "description": "single element matches, valid minContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "some elements match, valid minContains", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "all elements match, valid minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "minContains=2 with contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 2 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "some elements match, invalid minContains", + "data": [ 1, 2 ], + "valid": false + }, + { + "description": "all elements match, valid minContains (exactly as needed)", + "data": [ 1, 1 ], + "valid": true + }, + { + "description": "all elements match, valid minContains (more than needed)", + "data": [ 1, 1, 1 ], + "valid": true + }, + { + "description": "some elements match, valid minContains", + "data": [ 1, 2, 1 ], + "valid": true + } + ] + }, + { + "description": "minContains=2 with contains with a decimal value", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 2.0 + }, + "tests": [ + { + "description": "one element matches, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "both elements match, valid minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "maxContains = minContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 2, + "minContains": 2 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "all elements match, invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "all elements match, invalid maxContains", + "data": [ 1, 1, 1 ], + "valid": false + }, + { + "description": "all elements match, valid maxContains and minContains", + "data": [ 1, 1 ], + "valid": true + } + ] + }, + { + "description": "maxContains < minContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "maxContains": 1, + "minContains": 3 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": false + }, + { + "description": "invalid minContains", + "data": [ 1 ], + "valid": false + }, + { + "description": "invalid maxContains", + "data": [ 1, 1, 1 ], + "valid": false + }, + { + "description": "invalid maxContains and minContains", + "data": [ 1, 1 ], + "valid": false + } + ] + }, + { + "description": "minContains = 0", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": true + }, + { + "description": "minContains = 0 makes contains always pass", + "data": [ 2 ], + "valid": true + } + ] + }, + { + "description": "minContains = 0 with maxContains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"const": 1}, + "minContains": 0, + "maxContains": 1 + }, + "tests": [ + { + "description": "empty data", + "data": [ ], + "valid": true + }, + { + "description": "not more than maxContains", + "data": [ 1 ], + "valid": true + }, + { + "description": "too many", + "data": [ 1, 1 ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json new file mode 100644 index 000000000..9d6a8b6d2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minItems.json @@ -0,0 +1,50 @@ +[ + { + "description": "minItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minItems": 1 + }, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "", + "valid": true + } + ] + }, + { + "description": "minItems validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minItems": 1.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json new file mode 100644 index 000000000..5076c5a92 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minLength.json @@ -0,0 +1,55 @@ +[ + { + "description": "minLength validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minLength": 2 + }, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 1, + "valid": true + }, + { + "description": "one grapheme is not long enough", + "data": "\uD83D\uDCA9", + "valid": false + } + ] + }, + { + "description": "minLength validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minLength": 2.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json new file mode 100644 index 000000000..12587a5f8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minProperties.json @@ -0,0 +1,70 @@ +[ + { + "description": "minProperties validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minProperties": 1 + }, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "minProperties validation with a decimal", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minProperties": 1.0 + }, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json new file mode 100644 index 000000000..dc4405278 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/minimum.json @@ -0,0 +1,75 @@ +[ + { + "description": "minimum validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": 1.1 + }, + "tests": [ + { + "description": "above the minimum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 1.1, + "valid": true + }, + { + "description": "below the minimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "minimum validation with signed integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": -2 + }, + "tests": [ + { + "description": "negative above the minimum is valid", + "data": -1, + "valid": true + }, + { + "description": "positive above the minimum is valid", + "data": 0, + "valid": true + }, + { + "description": "boundary point is valid", + "data": -2, + "valid": true + }, + { + "description": "boundary point with float is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float below the minimum is invalid", + "data": -2.0001, + "valid": false + }, + { + "description": "int below the minimum is invalid", + "data": -3, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json new file mode 100644 index 000000000..76fa4765b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/multipleOf.json @@ -0,0 +1,102 @@ +[ + { + "description": "by int", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 2 + }, + "tests": [ + { + "description": "int by int", + "data": 10, + "valid": true + }, + { + "description": "int by int fail", + "data": 7, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "by number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 1.5 + }, + "tests": [ + { + "description": "zero is multiple of anything", + "data": 0, + "valid": true + }, + { + "description": "4.5 is multiple of 1.5", + "data": 4.5, + "valid": true + }, + { + "description": "-4.5 is multiple of 1.5", + "data": -4.5, + "valid": true + }, + { + "description": "35 is not multiple of 1.5", + "data": 35, + "valid": false + } + ] + }, + { + "description": "by small number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "multipleOf": 0.0001 + }, + "tests": [ + { + "description": "0.0075 is multiple of 0.0001", + "data": 0.0075, + "valid": true + }, + { + "description": "0.00751 is not multiple of 0.0001", + "data": 0.00751, + "valid": false + } + ] + }, + { + "description": "float division = inf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", "multipleOf": 0.123456789 + }, + "tests": [ + { + "description": "always invalid, but naive implementations may raise an overflow error", + "data": 1e308, + "valid": false + } + ] + }, + { + "description": "small multiple of large integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", "multipleOf": 1e-8 + }, + "tests": [ + { + "description": "any integer is a multiple of 1e-8", + "data": 12391239123, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json new file mode 100644 index 000000000..d0f2b6e84 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/not.json @@ -0,0 +1,301 @@ +[ + { + "description": "not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"type": "integer"} + }, + "tests": [ + { + "description": "allowed", + "data": "foo", + "valid": true + }, + { + "description": "disallowed", + "data": 1, + "valid": false + } + ] + }, + { + "description": "not multiple types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {"type": ["integer", "boolean"]} + }, + "tests": [ + { + "description": "valid", + "data": "foo", + "valid": true + }, + { + "description": "mismatch", + "data": 1, + "valid": false + }, + { + "description": "other mismatch", + "data": true, + "valid": false + } + ] + }, + { + "description": "not more complex schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { + "type": "object", + "properties": { + "foo": { + "type": "string" + } + } + } + }, + "tests": [ + { + "description": "match", + "data": 1, + "valid": true + }, + { + "description": "other match", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "mismatch", + "data": {"foo": "bar"}, + "valid": false + } + ] + }, + { + "description": "forbidden property", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "not": {} + } + } + }, + "tests": [ + { + "description": "property present", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "property absent", + "data": {"bar": 1, "baz": 2}, + "valid": true + } + ] + }, + { + "description": "forbid everything with empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": {} + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "forbid everything with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": true + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "allow everything with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": false + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "double negation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { "not": {} } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "collect annotations inside a 'not', even if collection is disabled", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "not": { + "$comment": "this subschema must still produce annotations internally, even though the 'not' will ultimately discard them", + "anyOf": [ + true, + { "properties": { "foo": true } } + ], + "unevaluatedProperties": false + } + }, + "tests": [ + { + "description": "unevaluated property", + "data": { "bar": 1 }, + "valid": true + }, + { + "description": "annotations are still collected inside a 'not'", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json new file mode 100644 index 000000000..7a7c7ffe3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/oneOf.json @@ -0,0 +1,293 @@ +[ + { + "description": "oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": 1, + "valid": true + }, + { + "description": "second oneOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both oneOf valid", + "data": 3, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "oneOf with base schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "oneOf" : [ + { + "minLength": 2 + }, + { + "maxLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one oneOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both oneOf valid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, true, true] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, one true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, false, false] + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "oneOf with boolean schemas, more than one true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [true, true, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [false, false, false] + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf complex types", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second oneOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both oneOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": false + }, + { + "description": "neither oneOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "oneOf with empty schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "one valid - valid", + "data": "foo", + "valid": true + }, + { + "description": "both valid - invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "oneOf with required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "oneOf": [ + { "required": ["foo", "bar"] }, + { "required": ["foo", "baz"] } + ] + }, + "tests": [ + { + "description": "both invalid - invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "first valid - valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "second valid - valid", + "data": {"foo": 1, "baz": 3}, + "valid": true + }, + { + "description": "both valid - invalid", + "data": {"foo": 1, "bar": 2, "baz" : 3}, + "valid": false + } + ] + }, + { + "description": "oneOf with missing optional property", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "properties": { + "bar": true, + "baz": true + }, + "required": ["bar"] + }, + { + "properties": { + "foo": true + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": {"bar": 8}, + "valid": true + }, + { + "description": "second oneOf valid", + "data": {"foo": "foo"}, + "valid": true + }, + { + "description": "both oneOf valid", + "data": {"foo": "foo", "bar": 8}, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": {"baz": "quux"}, + "valid": false + } + ] + }, + { + "description": "nested oneOf, to check validation semantics", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "oneOf": [ + { + "oneOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json new file mode 100644 index 000000000..6d6713be5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/anchor.json @@ -0,0 +1,60 @@ +[ + { + "description": "$anchor inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an $anchor buried in the enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "anchor_in_enum": { + "enum": [ + { + "$anchor": "my_anchor", + "type": "null" + } + ] + }, + "real_identifier_in_schema": { + "$anchor": "my_anchor", + "type": "string" + }, + "zzz_anchor_in_const": { + "const": { + "$anchor": "my_anchor", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/anchor_in_enum" }, + { "$ref": "#my_anchor" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$anchor": "my_anchor", + "type": "null" + }, + "valid": true + }, + { + "description": "in implementations that strip $anchor, this may match either $def", + "data": { + "type": "null" + }, + "valid": false + }, + { + "description": "match $ref to $anchor", + "data": "a string to match #/$defs/anchor_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to $anchor", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json new file mode 100644 index 000000000..d69b29e85 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/bignum.json @@ -0,0 +1,110 @@ +[ + { + "description": "integer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" + }, + "tests": [ + { + "description": "a bignum is an integer", + "data": 12345678910111213141516171819202122232425262728293031, + "valid": true + }, + { + "description": "a negative bignum is an integer", + "data": -12345678910111213141516171819202122232425262728293031, + "valid": true + } + ] + }, + { + "description": "number", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "number" + }, + "tests": [ + { + "description": "a bignum is a number", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": true + }, + { + "description": "a negative bignum is a number", + "data": -98249283749234923498293171823948729348710298301928331, + "valid": true + } + ] + }, + { + "description": "string", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" + }, + "tests": [ + { + "description": "a bignum is not a string", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": false + } + ] + }, + { + "description": "maximum integer comparison", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "maximum": 18446744073709551615 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMaximum": 972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 972783798187987123879878123.188781371, + "valid": false + } + ] + }, + { + "description": "minimum integer comparison", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "minimum": -18446744073709551615 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision on negative numbers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "exclusiveMinimum": -972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -972783798187987123879878123.188781371, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json new file mode 100644 index 000000000..5113bd649 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/cross-draft.json @@ -0,0 +1,18 @@ +[ + { + "description": "refs to historic drafts are processed as historic drafts", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "$ref": "http://localhost:1234/draft2019-09/ignore-prefixItems.json" + }, + "tests": [ + { + "description": "first item not a string is valid", + "comment": "if the implementation is not processing the $ref as a 2019-09 schema, this test will fail", + "data": [1, 2, 3], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json new file mode 100644 index 000000000..47d5bd79d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dependencies-compatibility.json @@ -0,0 +1,282 @@ +[ + { + "description": "single dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "empty dependents", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependents required", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo\nbar": ["foo\rbar"], + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "CRLF", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "quoted quotes", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "CRLF missing dependent", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "quoted quotes missing dependent", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + }, + { + "description": "single schema dependency", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "boolean subschemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "schema dependencies with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "dependencies": { + "foo\tbar": {"minProperties": 4}, + "foo'bar": {"required": ["foo\"bar"]} + } + }, + "tests": [ + { + "description": "quoted tab", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "quoted quote", + "data": { + "foo'bar": {"foo\"bar": 1} + }, + "valid": false + }, + { + "description": "quoted tab invalid under dependent schema", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "quoted quote invalid under dependent schema", + "data": {"foo'bar": 1}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json new file mode 100644 index 000000000..7e63f209a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/dynamicRef.json @@ -0,0 +1,56 @@ +[ + { + "description": "$dynamicRef skips over intermediate resources - pointer reference across resource boundary", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://test.json-schema.org/dynamic-ref-skips-intermediate-resource/optional/main", + "type": "object", + "properties": { + "bar-item": { + "$ref": "bar#/$defs/item" + } + }, + "$defs": { + "bar": { + "$id": "bar", + "type": "array", + "items": { + "$ref": "item" + }, + "$defs": { + "item": { + "$id": "item", + "type": "object", + "properties": { + "content": { + "$dynamicRef": "#content" + } + }, + "$defs": { + "defaultContent": { + "$dynamicAnchor": "content", + "type": "integer" + } + } + }, + "content": { + "$dynamicAnchor": "content", + "type": "string" + } + } + } + } + }, + "tests": [ + { + "description": "integer property passes", + "data": { "bar-item": { "content": 42 } }, + "valid": true + }, + { + "description": "string property fails", + "data": { "bar-item": { "content": "value" } }, + "valid": false + } + ] + }] \ No newline at end of file diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json new file mode 100644 index 000000000..a4d62e0cf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/ecmascript-regex.json @@ -0,0 +1,582 @@ +[ + { + "description": "ECMA 262 regex $ does not match trailing newline", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^abc$" + }, + "tests": [ + { + "description": "matches in Python, but not in ECMA 262", + "data": "abc\\n", + "valid": false + }, + { + "description": "matches", + "data": "abc", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex converts \\t to horizontal tab", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\t$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\t", + "valid": false + }, + { + "description": "matches", + "data": "\u0009", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and upper letter", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\cC$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cC", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and lower letter", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\cc$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cc", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\d matches ascii digits only", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\d$" + }, + "tests": [ + { + "description": "ASCII zero matches", + "data": "0", + "valid": true + }, + { + "description": "NKO DIGIT ZERO does not match (unlike e.g. Python)", + "data": "߀", + "valid": false + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) does not match", + "data": "\u07c0", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\D matches everything but ascii digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\D$" + }, + "tests": [ + { + "description": "ASCII zero does not match", + "data": "0", + "valid": false + }, + { + "description": "NKO DIGIT ZERO matches (unlike e.g. Python)", + "data": "߀", + "valid": true + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) matches", + "data": "\u07c0", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\w matches ascii letters only", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\w$" + }, + "tests": [ + { + "description": "ASCII 'a' matches", + "data": "a", + "valid": true + }, + { + "description": "latin-1 e-acute does not match (unlike e.g. Python)", + "data": "é", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\W matches everything but ascii letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\W$" + }, + "tests": [ + { + "description": "ASCII 'a' does not match", + "data": "a", + "valid": false + }, + { + "description": "latin-1 e-acute matches (unlike e.g. Python)", + "data": "é", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\s matches whitespace", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\s$" + }, + "tests": [ + { + "description": "ASCII space matches", + "data": " ", + "valid": true + }, + { + "description": "Character tabulation matches", + "data": "\t", + "valid": true + }, + { + "description": "Line tabulation matches", + "data": "\u000b", + "valid": true + }, + { + "description": "Form feed matches", + "data": "\u000c", + "valid": true + }, + { + "description": "latin-1 non-breaking-space matches", + "data": "\u00a0", + "valid": true + }, + { + "description": "zero-width whitespace matches", + "data": "\ufeff", + "valid": true + }, + { + "description": "line feed matches (line terminator)", + "data": "\u000a", + "valid": true + }, + { + "description": "paragraph separator matches (line terminator)", + "data": "\u2029", + "valid": true + }, + { + "description": "EM SPACE matches (Space_Separator)", + "data": "\u2003", + "valid": true + }, + { + "description": "Non-whitespace control does not match", + "data": "\u0001", + "valid": false + }, + { + "description": "Non-whitespace does not match", + "data": "\u2013", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\S matches everything but whitespace", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\S$" + }, + "tests": [ + { + "description": "ASCII space does not match", + "data": " ", + "valid": false + }, + { + "description": "Character tabulation does not match", + "data": "\t", + "valid": false + }, + { + "description": "Line tabulation does not match", + "data": "\u000b", + "valid": false + }, + { + "description": "Form feed does not match", + "data": "\u000c", + "valid": false + }, + { + "description": "latin-1 non-breaking-space does not match", + "data": "\u00a0", + "valid": false + }, + { + "description": "zero-width whitespace does not match", + "data": "\ufeff", + "valid": false + }, + { + "description": "line feed does not match (line terminator)", + "data": "\u000a", + "valid": false + }, + { + "description": "paragraph separator does not match (line terminator)", + "data": "\u2029", + "valid": false + }, + { + "description": "EM SPACE does not match (Space_Separator)", + "data": "\u2003", + "valid": false + }, + { + "description": "Non-whitespace control matches", + "data": "\u0001", + "valid": true + }, + { + "description": "Non-whitespace matches", + "data": "\u2013", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with pattern", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "\\p{Letter}cole" + }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "\\w in patterns matches [A-Za-z0-9_], not unicode letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "\\wcole" + }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "pattern with ASCII ranges", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "[a-z]cole" + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "ascii characters match", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + } + ] + }, + { + "description": "\\d in pattern matches [0-9], not unicode digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^\\d+$" + }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": false + } + ] + }, + { + "description": "pattern with non-ASCII digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^\\p{digit}+$" + }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "\\p{Letter}cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "\\w in patternProperties matches [A-Za-z0-9_], not unicode letters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "\\wcole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with ASCII ranges", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "[a-z]cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "ascii characters match", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + } + ] + }, + { + "description": "\\d in patternProperties matches [0-9], not unicode digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\d+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with non-ASCII digits", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\p{digit}+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json new file mode 100644 index 000000000..f5ae8b18f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/float-overflow.json @@ -0,0 +1,17 @@ +[ + { + "description": "all integers are multiples of 0.5, if overflow is handled", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer", + "multipleOf": 0.5 + }, + "tests": [ + { + "description": "valid if optional overflow handling is implemented", + "data": 1e308, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json new file mode 100644 index 000000000..96f26fefb --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format-assertion.json @@ -0,0 +1,44 @@ +[ + { + "description": "schema that uses custom metaschema with format-assertion: false", + "comment": "The true/false boolean in $vocabulary only controls behavior for implementations that do not recognize the vocabulary. For implementations that do understand format-assertion, the boolean has no impact on validation behavior — hence both test groups produce identical results.", + "schema": { + "$id": "https://schema/using/format-assertion/false", + "$schema": "http://localhost:1234/draft2020-12/format-assertion-false.json", + "format": "ipv4" + }, + "tests": [ + { + "description": "format-assertion: false: valid string", + "data": "127.0.0.1", + "valid": true + }, + { + "description": "format-assertion: false: invalid string", + "comment": "valid: false is intentional — the false value in $vocabulary only affects unknown vocabulary handling, not validation behavior itself", + "data": "not-an-ipv4", + "valid": false + } + ] + }, + { + "description": "schema that uses custom metaschema with format-assertion: true", + "schema": { + "$id": "https://schema/using/format-assertion/true", + "$schema": "http://localhost:1234/draft2020-12/format-assertion-true.json", + "format": "ipv4" + }, + "tests": [ + { + "description": "format-assertion: true: valid string", + "data": "127.0.0.1", + "valid": true + }, + { + "description": "format-assertion: true: invalid string", + "data": "not-an-ipv4", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json new file mode 100644 index 000000000..590f2f678 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date-time.json @@ -0,0 +1,250 @@ +[ + { + "description": "validation of date-time strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date-time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date-time string", + "data": "1963-06-19T08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid date-time string without second fraction", + "data": "1963-06-19T08:30:06Z", + "valid": true + }, + { + "description": "a valid date-time string with plus offset", + "data": "1937-01-01T12:00:27.87+00:20", + "valid": true + }, + { + "description": "a valid date-time string with minus offset", + "data": "1990-12-31T15:59:50.123-08:00", + "valid": true + }, + { + "description": "a valid date-time with a leap second, UTC", + "data": "1998-12-31T23:59:60Z", + "valid": true + }, + { + "description": "a valid date-time with a leap second, with minus offset", + "data": "1998-12-31T15:59:60.123-08:00", + "valid": true + }, + { + "description": "an invalid date-time past leap second, UTC", + "data": "1998-12-31T23:59:61Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong minute, UTC", + "data": "1998-12-31T23:58:60Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong hour, UTC", + "data": "1998-12-31T22:59:60Z", + "valid": false + }, + { + "description": "an invalid day in date-time string", + "data": "1990-02-31T15:59:59.123-08:00", + "valid": false + }, + { + "description": "an invalid offset in date-time string", + "data": "1990-12-31T15:59:59-24:00", + "valid": false + }, + { + "description": "an invalid closing Z after time-zone offset", + "data": "1963-06-19T08:30:06.28123+01:00Z", + "valid": false + }, + { + "description": "an invalid hour in date-time string", + "data": "1990-12-31T24:00:00Z", + "valid": false + }, + { + "description": "an invalid minute in date-time string", + "data": "1990-12-31T15:60:00Z", + "valid": false + }, + { + "description": "an invalid offset minute in date-time string", + "data": "1990-12-31T10:00:00+10:60", + "valid": false + }, + { + "description": "an invalid date-time string", + "data": "06/19/1963 08:30:06 PST", + "valid": false + }, + { + "description": "case-insensitive T and Z", + "data": "1963-06-19t08:30:06.283185z", + "valid": true + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350T01:01:01", + "valid": false + }, + { + "description": "invalid non-padded month dates", + "data": "1963-6-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-padded day dates", + "data": "1963-06-1T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in date portion", + "data": "1963-06-1৪T00:00:00Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in time portion", + "data": "1963-06-11T0৪:00:00Z", + "valid": false + }, + { + "description": "invalid extended year", + "data": "+11963-06-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "a numeric offset without minutes is invalid", + "data": "1985-04-12T23:20:50+01", + "valid": false + }, + { + "description": "hour 24 is invalid even with a leap second", + "data": "2016-12-31T24:59:60+01:00", + "valid": false + }, + { + "description": "a second fraction of fifteen nines is valid", + "data": "1985-04-12T00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a trailing newline is invalid", + "data": "1985-04-12T23:20:50Z\n", + "valid": false + }, + { + "description": "an invalid date-time string without seconds", + "data": "1985-04-12T23:20Z", + "valid": false + }, + { + "description": "an invalid date-time string with trailing content after the offset", + "data": "1985-04-12T23:20:50Ztail", + "valid": false + }, + { + "description": "an invalid minute in date-time string, with a numeric offset", + "data": "1985-04-12T23:60:00+00:01", + "valid": false + }, + { + "description": "a valid date-time string with 28 days in February (normal)", + "data": "2021-02-28T00:00:00Z", + "valid": true + }, + { + "description": "an invalid date-time string with 30 days in February (leap)", + "data": "2020-02-30T00:00:00Z", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29T00:00:00Z", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 0100 is not a leap year", + "data": "0100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "data": "0400-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "data": "2100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "1985-04-12T23:20:50.Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "1985-04-12T23:20:50,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": " 1985-04-12T23:20:50Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": "1985-04-12T23:20:50Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json new file mode 100644 index 000000000..a8a19261a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/date.json @@ -0,0 +1,423 @@ +[ + { + "description": "validation of date strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "date" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date string", + "data": "1963-06-19", + "valid": true + }, + { + "description": "a valid date string with 31 days in January", + "data": "2020-01-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in January", + "data": "2020-01-32", + "valid": false + }, + { + "description": "a valid date string with 28 days in February (normal)", + "data": "2021-02-28", + "valid": true + }, + { + "description": "a invalid date string with 30 days in February (leap)", + "data": "2020-02-30", + "valid": false + }, + { + "description": "a valid date string with 31 days in March", + "data": "2020-03-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in March", + "data": "2020-03-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in April", + "data": "2020-04-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in April", + "data": "2020-04-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in May", + "data": "2020-05-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in May", + "data": "2020-05-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in June", + "data": "2020-06-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in June", + "data": "2020-06-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in July", + "data": "2020-07-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in July", + "data": "2020-07-32", + "valid": false + }, + { + "description": "a valid date string with 31 days in August", + "data": "2020-08-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in August", + "data": "2020-08-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in September", + "data": "2020-09-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in September", + "data": "2020-09-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in October", + "data": "2020-10-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in October", + "data": "2020-10-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in November", + "data": "2020-11-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in November", + "data": "2020-11-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in December", + "data": "2020-12-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in December", + "data": "2020-12-32", + "valid": false + }, + { + "description": "an invalid date string", + "data": "06/19/1963", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350", + "valid": false + }, + { + "description": "non-padded month dates are not valid", + "data": "1998-1-20", + "valid": false + }, + { + "description": "non-padded day dates are not valid", + "data": "1998-01-1", + "valid": false + }, + { + "description": "invalid month", + "data": "1998-13-01", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29", + "valid": true + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1963-06-1৪", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in month field", + "data": "2020-0৪-01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: YYYYMMDD without dashes (2023-03-28)", + "data": "20230328", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number implicit day of week (2023-01-02)", + "data": "2023-W01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number with day of week (2023-03-28)", + "data": "2023-W13-2", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number rollover to next year (2023-01-01)", + "data": "2022W527", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "century year 0100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 100 % 100 == 0 but 100 % 400 != 0", + "data": "0100-02-29", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 400-year cycle boundary", + "data": "0400-02-29", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — future century year", + "data": "2100-02-29", + "valid": false + }, + { + "description": "invalid: leading whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": " 2024-01-15", + "valid": false + }, + { + "description": "invalid: trailing whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": "2024-01-15 ", + "valid": false + }, + { + "description": "invalid: month 00 is not valid per date-month range 01-12", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-month = 2DIGIT ; 01-12", + "data": "2024-00-15", + "valid": false + }, + { + "description": "invalid: day 00 is not valid per date-mday minimum of 01", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-mday = 2DIGIT ; 01-28/29/30/31", + "data": "2024-01-00", + "valid": false + }, + { + "description": "invalid: empty string", + "data": "", + "valid": false + }, + { + "description": "invalid: embedded whitespace between year and month", + "data": "2020 -01-01", + "valid": false + }, + { + "description": "invalid: trailing character after valid full-date", + "data": "2020-01-01X", + "valid": false + }, + { + "description": "invalid: trailing Z after full-date", + "data": "2020-01-01Z", + "valid": false + }, + { + "description": "invalid: full-date followed by space and time component", + "data": "2020-01-01 00:00:00Z", + "valid": false + }, + { + "description": "valid: four-digit year 0001", + "data": "0001-01-01", + "valid": true + }, + { + "description": "invalid: two-digit year (N-2 digits)", + "data": "20-01-01", + "valid": false + }, + { + "description": "invalid: three-digit year (N-1 digits)", + "data": "998-01-01", + "valid": false + }, + { + "description": "invalid: five-digit year (N+1 digits)", + "data": "12020-01-01", + "valid": false + }, + { + "description": "invalid: positive sign prefix on year", + "data": "+2020-01-01", + "valid": false + }, + { + "description": "invalid: negative sign prefix on year", + "data": "-2020-01-01", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in year field", + "data": "২020-01-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in year field", + "data": "YYYY-01-01", + "valid": false + }, + { + "description": "invalid: three-digit month (N+1 digits)", + "data": "2020-001-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in month field", + "data": "2020-MM-01", + "valid": false + }, + { + "description": "invalid: three-digit day (N+1 digits)", + "data": "2020-01-001", + "valid": false + }, + { + "description": "invalid: alphabetic characters in day field", + "data": "2020-01-DD", + "valid": false + }, + { + "description": "invalid: colon separators", + "data": "2020:01:01", + "valid": false + }, + { + "description": "invalid: dot separators", + "data": "2020.01.01", + "valid": false + }, + { + "description": "invalid: space separators", + "data": "2020 01 01", + "valid": false + }, + { + "description": "invalid: mixed slash and hyphen separators", + "data": "2020-01/01", + "valid": false + }, + { + "description": "invalid: duplicated first hyphen", + "data": "2020--01-01", + "valid": false + }, + { + "description": "invalid: duplicated second hyphen", + "data": "2020-01--01", + "valid": false + }, + { + "description": "valid: date inside the Julian to Gregorian reform gap", + "data": "1582-10-10", + "valid": true + }, + { + "description": "invalid: year field numerically larger than a 32-bit signed integer", + "data": "2147483648-01-01", + "valid": false + }, + { + "description": "invalid: character just above '9' as the second digit of the day", + "data": "2020-01-0:", + "valid": false + }, + { + "description": "invalid: non-ASCII digit leaving ten UTF-8 bytes but eight characters", + "data": "\u09e80-01-01", + "valid": false + }, + { + "description": "invalid: en dash separators instead of hyphen-minus", + "data": "2020\u201301\u201301", + "valid": false + }, + { + "description": "invalid: NUL character after an otherwise valid full-date", + "data": "2020-01-01\u0000", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json new file mode 100644 index 000000000..e3e184161 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/duration.json @@ -0,0 +1,268 @@ +[ + { + "description": "validation of duration strings", + "comment": "RFC 3339 Appendix A defines the ABNF grammar for ISO-8601 durations used by JSON Schema format 'duration'. These tests enforce only the syntax defined by that grammar.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "duration" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid duration string", + "data": "P4DT12H30M5S", + "valid": true + }, + { + "description": "an invalid duration string", + "data": "PT1D", + "valid": false + }, + { + "description": "must start with P", + "data": "4DT12H30M5S", + "valid": false + }, + { + "description": "no elements present", + "data": "P", + "valid": false + }, + { + "description": "no time elements present", + "data": "P1YT", + "valid": false + }, + { + "description": "no date or time elements present", + "data": "PT", + "valid": false + }, + { + "description": "elements out of order", + "data": "P2D1Y", + "valid": false + }, + { + "description": "missing time separator", + "data": "P1D2H", + "valid": false + }, + { + "description": "time element in the date position", + "data": "P2S", + "valid": false + }, + { + "description": "four years duration", + "data": "P4Y", + "valid": true + }, + { + "description": "zero time, in seconds", + "data": "PT0S", + "valid": true + }, + { + "description": "zero time, in days", + "data": "P0D", + "valid": true + }, + { + "description": "one month duration", + "data": "P1M", + "valid": true + }, + { + "description": "one minute duration", + "data": "PT1M", + "valid": true + }, + { + "description": "one and a half days, in hours", + "data": "PT36H", + "valid": true + }, + { + "description": "one and a half days, in days and hours", + "data": "P1DT12H", + "valid": true + }, + { + "description": "two weeks", + "data": "P2W", + "valid": true + }, + { + "description": "weeks cannot be combined with other units", + "data": "P1Y2W", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "P২Y", + "valid": false + }, + { + "description": "element without unit", + "data": "P1", + "valid": false + }, + { + "description": "all date and time components", + "data": "P1Y2M3DT4H5M6S", + "valid": true + }, + { + "description": "date components only", + "data": "P1Y2M3D", + "valid": true + }, + { + "description": "time components only", + "data": "PT1H2M3S", + "valid": true + }, + { + "description": "month and day", + "data": "P1M2D", + "valid": true + }, + { + "description": "hour and minute", + "data": "PT1H30M", + "valid": true + }, + { + "description": "multi-digit values in all components", + "data": "P10Y10M10DT10H10M10S", + "valid": true + }, + { + "description": "fractional duration is not allowed by RFC 3339 ABNF", + "comment": "numeric components use 1*DIGIT where DIGIT = %x30-39; '.' is not allowed", + "data": "PT0.5S", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " P1D", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "P1D ", + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "years and months can appear without days", + "data": "P1Y2M", + "valid": true + }, + { + "description": "years and days cannot appear without months", + "data": "P1Y2D", + "valid": false + }, + { + "description": "hours and minutes can appear without seconds", + "data": "PT1H2M", + "valid": true + }, + { + "description": "hours and seconds cannot appear without minutes", + "data": "PT1H2S", + "valid": false + }, + { + "description": "minutes and seconds can appear without hour", + "data": "PT1M2S", + "valid": true + }, + { + "description": "a leading sign is not allowed", + "data": "-P1D", + "valid": false + }, + { + "description": "a number before the time separator has no unit", + "data": "P1D2T3H", + "valid": false + }, + { + "description": "exponent notation is not allowed in a component", + "data": "P1e2D", + "valid": false + }, + { + "description": "a trailing newline is invalid", + "data": "P1D\n", + "valid": false + }, + { + "description": "weeks cannot be combined with a time component", + "data": "P1WT1H", + "valid": false + }, + { + "description": "weeks cannot be combined with a zero-valued component", + "data": "P0Y1W", + "valid": false + }, + { + "description": "a leading zero in a component is valid", + "data": "P01D", + "valid": true + }, + { + "description": "a component with many digits is valid", + "data": "P999999999999999999999999999999999999999999999999999999999999999999999999999999D", + "valid": true + }, + { + "description": "a comma as the decimal separator is invalid", + "data": "PT0,5S", + "valid": false + }, + { + "description": "a sign inside a component is invalid", + "data": "P-1D", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json new file mode 100644 index 000000000..b9ae3c049 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ecmascript-regex.json @@ -0,0 +1,116 @@ +[ + { + "description": "\\a is not an ECMA 262 control escape", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "when used as a pattern", + "data": "\\a", + "valid": false + } + ] + }, + { + "description": "Python-specific regular expression syntax is not valid ECMA 262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "Python named group (?P...) is not ECMA 262", + "data": "(?Px)", + "valid": false + }, + { + "description": "Python named backreference (?P=name) is not ECMA 262", + "data": "(?Pa)(?P=n)", + "valid": false + }, + { + "description": "inline comment group (?#...) is not ECMA 262", + "data": "(?#comment)a", + "valid": false + } + ] + }, + { + "description": "global inline flag groups are not valid ECMA 262", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "a single global inline flag (?i)", + "data": "(?i)abc", + "valid": false + }, + { + "description": "multiple global inline flags (?ims)", + "data": "(?ims)abc", + "valid": false + } + ] + }, + { + "description": "ECMA 262 named groups and backreferences are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "an ECMA 262 named group (?...)", + "data": "(?x)", + "valid": true + }, + { + "description": "an ECMA 262 named backreference \\k", + "data": "(?a)\\k", + "valid": true + } + ] + }, + { + "description": "ECMA 262 lookbehind is valid, including variable width", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "a variable-width lookbehind (ES2018)", + "data": "(?<=a+)b", + "valid": true + } + ] + }, + { + "description": "ECMA 262 character classes and escapes", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "an empty character class is valid ECMA 262", + "data": "[]", + "valid": true + }, + { + "description": "a negated empty character class is valid ECMA 262", + "data": "[^]", + "valid": true + }, + { + "description": "a control escape \\cA is valid ECMA 262", + "data": "\\cA", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json new file mode 100644 index 000000000..50e214622 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/email.json @@ -0,0 +1,368 @@ +[ + { + "description": "validation of e-mail addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "an invalid e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "tilde in local part is valid", + "data": "te~st@example.com", + "valid": true + }, + { + "description": "tilde before local part is valid", + "data": "~test@example.com", + "valid": true + }, + { + "description": "tilde after local part is valid", + "data": "test~@example.com", + "valid": true + }, + { + "description": "a quoted string with a space in the local part is valid", + "data": "\"joe bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a double dot in the local part is valid", + "data": "\"joe..bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a @ in the local part is valid", + "data": "\"joe@bloggs\"@example.com", + "valid": true + }, + { + "description": "an IPv4-address-literal after the @ is valid", + "data": "joe.bloggs@[127.0.0.1]", + "valid": true + }, + { + "description": "an IPv6-address-literal after the @ is valid", + "data": "joe.bloggs@[IPv6:::1]", + "valid": true + }, + { + "description": "dot before local part is not valid", + "data": ".test@example.com", + "valid": false + }, + { + "description": "dot after local part is not valid", + "data": "test.@example.com", + "valid": false + }, + { + "description": "two separated dots inside local part are valid", + "data": "te.s.t@example.com", + "valid": true + }, + { + "description": "two subsequent dots inside local part are not valid", + "data": "te..st@example.com", + "valid": false + }, + { + "description": "an invalid domain", + "data": "joe.bloggs@invalid=domain.com", + "valid": false + }, + { + "description": "an invalid IPv4-address-literal", + "data": "joe.bloggs@[127.0.0.300]", + "valid": false + }, + { + "description": "two email addresses is not valid", + "data": "user1@oceania.org, user2@oceania.org", + "valid": false + }, + { + "description": "full \"From\" header is invalid", + "data": "\"Winston Smith\" (Records Department)", + "valid": false + }, + { + "description": "local part is required", + "data": "@example.com", + "valid": false + }, + { + "description": "domain is required", + "data": "joe.bloggs@", + "valid": false + }, + { + "description": "unquoted space in local part is invalid", + "data": "joe bloggs@example.com", + "valid": false + }, + { + "description": "a domain that looks like an IPv4 address without brackets is valid", + "data": "test@255.255.255.255", + "valid": true + }, + { + "description": "an empty quoted string in the local part is valid", + "data": "\"\"@iana.org", + "valid": true + }, + { + "description": "a quoted string containing only a space in the local part is valid", + "data": "\" \"@iana.org", + "valid": true + }, + { + "description": "consecutive hyphens inside a domain label are valid", + "data": "test@c--n.com", + "valid": true + }, + { + "description": "a domain label starting with a digit is valid", + "data": "test@123.com", + "valid": true + }, + { + "description": "a non-ASCII character in the local part is not valid", + "data": "a\u00e9@iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in the domain is not valid", + "data": "a@\u00e9.org", + "valid": false + }, + { + "description": "a fullwidth commercial at is not a valid separator", + "data": "a\uff20iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in a quoted pair is not valid", + "data": "\"test\\\u00a9\"@iana.org", + "valid": false + }, + { + "description": "a trailing line feed is not valid", + "data": "test@iana.org\n", + "valid": false + }, + { + "description": "a trailing carriage return is not valid", + "data": "test@iana.org\r", + "valid": false + }, + { + "description": "a trailing space is not valid", + "data": "a@iana.org ", + "valid": false + }, + { + "description": "a leading space is not valid", + "data": " a@iana.org", + "valid": false + }, + { + "description": "a tab in the local part is not valid", + "data": "a\tb@iana.org", + "valid": false + }, + { + "description": "a delete character in the local part is not valid", + "data": "\u007f@iana.org", + "valid": false + }, + { + "description": "a quoted string with no special characters in the local part is valid", + "data": "\"test\"@iana.org", + "valid": true + }, + { + "description": "a quoted pair in the local part is valid", + "data": "\"\\a\"@iana.org", + "valid": true + }, + { + "description": "an escaped double quote in the local part is valid", + "data": "\"\\\"\"@iana.org", + "valid": true + }, + { + "description": "an escaped backslash in the local part is valid", + "data": "\"\\\\\"@iana.org", + "valid": true + }, + { + "description": "an unescaped double quote in an unquoted local part is not valid", + "data": "test\"@iana.org", + "valid": false + }, + { + "description": "text after the closing double quote in the local part is not valid", + "data": "\"test\"test@iana.org", + "valid": false + }, + { + "description": "an unclosed double quote in the local part is not valid", + "data": "\"test@iana.org", + "valid": false + }, + { + "description": "two unquoted @ signs are not valid", + "data": "a@b@c.org", + "valid": false + }, + { + "description": "a backslash-escaped @ in an unquoted local part is not valid", + "data": "test\\@test@iana.org", + "valid": false + }, + { + "description": "an address literal in the local part position is not valid", + "data": "[1.2.3.4]@iana.org", + "valid": false + }, + { + "description": "an empty string is not valid", + "data": "", + "valid": false + }, + { + "description": "a domain label starting with a hyphen is not valid", + "data": "test@-iana.org", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "data": "test@iana-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "data": "test@.iana.org", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "test@iana.org.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "test@iana..com", + "valid": false + }, + { + "description": "a single-label domain is valid", + "data": "test@io", + "valid": true + }, + { + "description": "an unclosed address literal is not valid", + "data": "test@[1.2.3.4", + "valid": false + }, + { + "description": "empty brackets after the @ are not valid", + "data": "a@[]", + "valid": false + }, + { + "description": "an IPv4-address-literal with three octets is not valid", + "data": "a@[1.2.3]", + "valid": false + }, + { + "description": "an IPv4-address-literal with five octets is not valid", + "data": "a@[1.2.3.4.5]", + "valid": false + }, + { + "description": "bracketed content that is not an address is not valid", + "data": "test@[RFC-5322-domain-literal]", + "valid": false + }, + { + "description": "a domain label before the opening bracket is not valid", + "data": "test@a[255.255.255.255]", + "valid": false + }, + { + "description": "a lowercase IPv6 tag in an address literal is valid", + "data": "a@[ipv6:::1]", + "valid": true + }, + { + "description": "a comma in an unquoted local part is not valid", + "data": "a,b@iana.org", + "valid": false + }, + { + "description": "a semicolon in an unquoted local part is not valid", + "data": "a;b@iana.org", + "valid": false + }, + { + "description": "a colon in an unquoted local part is not valid", + "data": "a:b@iana.org", + "valid": false + }, + { + "description": "a parenthesis in an unquoted local part is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "test(comment)@iana.org", + "valid": false + }, + { + "description": "a parenthesis in the domain is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "a@b(c).org", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json new file mode 100644 index 000000000..7cacfcfd2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/hostname.json @@ -0,0 +1,386 @@ +[ + { + "description": "validation of host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name", + "data": "www.example.com", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "IDN label separator", + "data": "example\uff0ecom", + "valid": false + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "starts with hyphen", + "data": "-hostname", + "valid": false + }, + { + "description": "ends with hyphen", + "data": "hostname-", + "valid": false + }, + { + "description": "contains underscore", + "data": "host_name", + "valid": false + }, + { + "description": "exceeds maximum overall length (256)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": false + }, + { + "description": "maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": true + }, + { + "description": "exceeds maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl.com", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "invalid non-ASCII KELVIN SIGN (U+212A)", + "data": "\u212Aelvin.example.com", + "valid": false + }, + { + "description": "consecutive hyphens inside a label", + "data": "a--b.com", + "valid": true + } + ] + }, + { + "description": "validation of A-label (punycode) host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "hostname" + }, + "tests": [ + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "xn--9n2bp8q.xn--9t4b11yi5a", + "valid": true + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "xn--07jt112bpxg.xn--9t4b11yi5a", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-txk", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-zed", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-6bf", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--zca29lwxobi7a", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--qmbc", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--chb89f", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "xn--07jceefgh4c", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--al-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-fda", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--la-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-gda", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--ll-0ea", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--S-jib3p", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3j", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3je", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--A-2hc5h", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--5db1e", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--4dbc5h", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--A-2hc8h", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--5db3e", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--4dbc8h", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--defabc-k64e", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--k8j5u", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--bck0j", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek778f", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngb6iyr", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngba1o", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "xn--0-gyc", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2er09f", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--02b508i", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2ezcw70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "xn--11b2ezcs70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "xn--ngba5hb2804a", + "valid": true + }, + { + "description": "contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json new file mode 100644 index 000000000..f46238b6f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-email.json @@ -0,0 +1,140 @@ +[ + { + "description": "validation of an internationalized e-mail addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-email" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid idn e-mail (example@example.test in Hangul)", + "data": "실례@실례.테스트", + "valid": true + }, + { + "description": "an invalid e-mail/idn e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "a non-ASCII local part with an ASCII domain is valid", + "data": "δοκιμή@example.com", + "valid": true + }, + { + "description": "a non-ASCII quoted local part is valid", + "data": "\"δοκιμή\"@example.com", + "valid": true + }, + { + "description": "a domain label that is not in Unicode NFC is valid", + "data": "user@cafe\u0301.com", + "valid": true + }, + { + "description": "a local part that is not in Unicode NFC is valid", + "data": "cafe\u0301@example.com", + "valid": true + }, + { + "description": "a C1 control in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-2 admits U+0085", + "data": "\u0085@example.com", + "valid": true + }, + { + "description": "a noncharacter in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-3 admits U+FFFF", + "data": "\uffff@example.com", + "valid": true + }, + { + "description": "a local part at the 64-octet limit is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.5.3.1 every implementation must be able to receive a local part of 64 octets", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@example.com", + "valid": true + }, + { + "description": "a fullwidth commercial at is not a local-part separator", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal ) splits on U+0040, and any IDN mapping applies only to the domain that split produces, so U+FF20 cannot become the delimiter", + "data": "user\uff20example.com", + "valid": false + }, + { + "description": "a local part with a supplementary-plane character is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii, and https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-4 covers U+10000 and above", + "data": "\ud835\udd4f@example.com", + "valid": true + }, + { + "description": "a domain label starting with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 sub-domain = Let-dig [Ldh-str], so a label starts with a letter or digit; RFC 6531 section 3.3 only adds sub-domain =/ U-label", + "data": "δοκιμή@-example.com", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig, so a label ends with a letter or digit", + "data": "δοκιμή@example-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Domain = sub-domain *(\".\" sub-domain) admits no empty sub-domain", + "data": "δοκιμή@.example.com", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "δοκιμή@example.com.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "δοκιμή@example..com", + "valid": false + }, + { + "description": "domain is required", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal )", + "data": "δοκιμή@", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json new file mode 100644 index 000000000..62f914d02 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/idn-hostname.json @@ -0,0 +1,588 @@ +[ + { + "description": "validation of internationalized host names", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "실례.테스트", + "valid": true + }, + { + "description": "illegal first char U+302E Hangul single dot tone mark", + "data": "〮실례.테스트", + "valid": false + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "실〮례.테스트", + "valid": false + }, + { + "description": "a single label of 63 characters is valid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": true + }, + { + "description": "a single label of 64 characters is too long", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "invalid label, correct Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc3492#section-7.1", + "data": "-> $1.00 <--", + "valid": false + }, + { + "description": "valid Chinese Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4", + "data": "xn--ihqwcrb4cv8a8dqg056pqjye", + "valid": true + }, + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "U-label contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + }, + { + "description": "U-label starts with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello", + "valid": false + }, + { + "description": "U-label ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "hello-", + "valid": false + }, + { + "description": "U-label starts and ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello-", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0903hello", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0300hello", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0488hello", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u00df\u03c2\u0f0b\u3007", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u06fd\u06fe", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u0640\u07fa", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "\u3031\u3032\u3033\u3034\u3035\u302e\u302f\u303b", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "a\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7a", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7l", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375S", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375\u03b2", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "A\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05d0\u05f3\u05d1", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "A\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05d0\u05f4\u05d1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "def\u30fbabc", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u3041", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u30a1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u4e08", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u06f0", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u0628", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "\u06f00", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u094d\u200d\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "\u0628\u064a\u200c\u0628\u064a", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "fullwidth digits are mapped to ASCII digits", + "comment": "IdnaMappingTable.txt maps U+FF11 to 0031, so this label maps to 123", + "data": "\uff11\uff12\uff13", + "valid": true + }, + { + "description": "a zero width space is ignored by the mapping", + "comment": "IdnaMappingTable.txt lists U+200B as ignored, so this label maps to ab", + "data": "a\u200bb", + "valid": true + }, + { + "description": "an A-label with an uppercase Punycode body is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.3 lowercases the A-label before the round-trip test, and RFC 3492 section 5 gives A-Z and a-z the same digit-values", + "data": "xn--NXASMQ6B", + "valid": true + }, + { + "description": "the two Arabic-Indic digit blocks may appear in different labels", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.8 scopes the rule to \"this label\", so one block per label is valid", + "data": "\u0628\u0660.\u0628\u06f0", + "valid": true + }, + { + "description": "a label of only Arabic-Indic digits is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 condition 1 allows only L, R or AL as the first character, and an Arabic-Indic digit is AN", + "data": "\u0660\u0661", + "valid": false + }, + { + "description": "a non-NFC U-label is valid after UTS 46 normalisation", + "comment": "UTS 46 section 4 normalises to NFC before validating, so the decomposed form is accepted", + "data": "cafe\u0301.com", + "valid": true + }, + { + "description": "a label of 63 fullwidth characters is valid after mapping", + "comment": "UTS 46 maps each U+FF41 to an ASCII letter, so this label is 63 octets in its mapped form", + "data": "\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41", + "valid": true + }, + { + "description": "an ACE prefix produced by the mapping step is decoded as an A-label", + "comment": "UTS 46 maps U+FF58 U+FF4E to \"xn\", so the mapped label is the A-label xn--nxasmq6b", + "data": "\uff58\uff4e--nxasmq6b", + "valid": true + }, + { + "description": "zero width non-joiner must pass at every occurrence", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937x\u200cy", + "valid": false + }, + { + "description": "Bidi domain name with a digit-first label is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 a label in a Bidi domain name must start with an L, R or AL character", + "data": "0a.\u05d0", + "valid": false + }, + { + "description": "label starting with a digit before a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "0\u0627", + "valid": false + }, + { + "description": "left-to-right label containing a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "a\u05d0", + "valid": false + }, + { + "description": "right-to-left label mixing both digit types is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "\u05d00\u0660", + "valid": false + }, + { + "description": "A-label that decodes to a disallowed code point is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5892#section-2.6", + "data": "xn--7a", + "valid": false + }, + { + "description": "A-label that decodes to a Bidi rule violation is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5893", + "data": "xn--0ca24w", + "valid": false + }, + { + "description": "a U-label whose A-label form is longer than 63 octets is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-4.2.4 the 63-octet limit is on the A-label form, not the code point count", + "data": "\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc", + "valid": false + }, + { + "description": "empty label between two dots is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc1034#section-3.1", + "data": "a..b", + "valid": false + }, + { + "description": "a name longer than 253 characters is invalid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "A-label that decodes to only ASCII is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 a U-label must have at least one non-ASCII character", + "data": "xn--example-", + "valid": false + }, + { + "description": "non-canonical Punycode that does not re-encode to itself is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.4 a label decoded from Punycode must be identical to the original A-label", + "data": "xn---9uc", + "valid": false + } + ] + }, + { + "description": "validation of separators in internationalized host names", + "specification": [ + {"rfc3490": "3.1", "quote": "Whenever dots are used as label separators, the following characters MUST be recognized as dots: U+002E (full stop), U+3002 (ideographic full stop), U+FF0E (fullwidth full stop), U+FF61(halfwidth ideographic full stop)"} + ], + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "idn-hostname" + }, + "tests": [ + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "single ideographic full stop", + "data": "\u3002", + "valid": false + }, + { + "description": "single fullwidth full stop", + "data": "\uff0e", + "valid": false + }, + { + "description": "single halfwidth ideographic full stop", + "data": "\uff61", + "valid": false + }, + { + "description": "dot as label separator", + "data": "a.b", + "valid": true + }, + { + "description": "ideographic full stop as label separator", + "data": "a\u3002b", + "valid": true + }, + { + "description": "fullwidth full stop as label separator", + "data": "a\uff0eb", + "valid": true + }, + { + "description": "halfwidth ideographic full stop as label separator", + "data": "a\uff61b", + "valid": true + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "leading ideographic full stop", + "data": "\u3002example", + "valid": false + }, + { + "description": "leading fullwidth full stop", + "data": "\uff0eexample", + "valid": false + }, + { + "description": "leading halfwidth ideographic full stop", + "data": "\uff61example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "trailing ideographic full stop", + "data": "example\u3002", + "valid": false + }, + { + "description": "trailing fullwidth full stop", + "data": "example\uff0e", + "valid": false + }, + { + "description": "trailing halfwidth ideographic full stop", + "data": "example\uff61", + "valid": false + }, + { + "description": "label too long if separator ignored (full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα.com", + "valid": true + }, + { + "description": "label too long if separator ignored (ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\u3002com", + "valid": true + }, + { + "description": "label too long if separator ignored (fullwidth full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff0ecom", + "valid": true + }, + { + "description": "label too long if separator ignored (halfwidth ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff61com", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json new file mode 100644 index 000000000..696713e90 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv4.json @@ -0,0 +1,217 @@ +[ + { + "description": "validation of IP addresses", + "comment": "RFC 2673, Section 3.2: dotted-quad = decbyte \".\" decbyte \".\" decbyte \".\" decbyte. A 'decbyte' (1*3DIGIT) restricts semantic values to 0-255, allows leading zeros, and strictly forbids symbols, alpha/hex, whitespace, and non-ASCII characters.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv4" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IP address", + "data": "192.168.0.1", + "valid": true + }, + { + "description": "an IP address with too many components", + "data": "127.0.0.0.1", + "valid": false + }, + { + "description": "an IP address with out-of-range values", + "data": "256.256.256.256", + "valid": false + }, + { + "description": "an IP address without 4 components", + "data": "127.0", + "valid": false + }, + { + "description": "a 2-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.1", + "valid": false + }, + { + "description": "a 3-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.0.1", + "valid": false + }, + { + "description": "an IP address as an integer", + "data": "0x7f000001", + "valid": false + }, + { + "description": "an IP address as an integer (decimal)", + "data": "2130706433", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২7.0.0.1", + "valid": false + }, + { + "description": "invalid fullwidth digits (non-ASCII)", + "data": "１９２.１６８.１.１", + "valid": false + }, + { + "description": "invalid mathematical bold digits (non-ASCII)", + "data": "𝟏𝟗𝟐.𝟏𝟔𝟖.𝟏.𝟏", + "valid": false + }, + { + "description": "netmask is not a part of ipv4 address", + "data": "192.168.1.0/24", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " 192.168.0.1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "192.168.0.1 ", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "192.168.0.1\n", + "valid": false + }, + { + "description": "hexadecimal notation is invalid", + "data": "0x7f.0.0.1", + "valid": false + }, + { + "description": "octal notation explicit is invalid", + "data": "0o10.0.0.1", + "valid": false + }, + { + "description": "empty part (double dot) is invalid", + "data": "192.168..1", + "valid": false + }, + { + "description": "leading dot is invalid", + "data": ".192.168.0.1", + "valid": false + }, + { + "description": "trailing dot is invalid", + "data": "192.168.0.1.", + "valid": false + }, + { + "description": "minimum valid IPv4 address", + "data": "0.0.0.0", + "valid": true + }, + { + "description": "maximum valid IPv4 address", + "data": "255.255.255.255", + "valid": true + }, + { + "description": "valid IPv4 with an octet in the 200 to 249 range", + "data": "200.0.0.0", + "valid": true + }, + { + "description": "valid IPv4 with two-digit octets", + "data": "10.20.30.40", + "valid": true + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "plus sign is invalid", + "data": "+1.2.3.4", + "valid": false + }, + { + "description": "negative sign is invalid", + "data": "-1.2.3.4", + "valid": false + }, + { + "description": "exponential notation is invalid", + "data": "1e2.0.0.1", + "valid": false + }, + { + "description": "alpha characters are invalid", + "data": "192.168.a.1", + "valid": false + }, + { + "description": "internal whitespace is invalid", + "data": "192. 168.0.1", + "valid": false + }, + { + "description": "tab character is invalid", + "data": "192.168.0.1\t", + "valid": false + }, + { + "description": "additional content after an embedded NUL byte", + "data": "192.168.0.1\u0000.evil.com", + "valid": false + }, + { + "description": "with port number is invalid", + "data": "192.168.0.1:80", + "valid": false + }, + { + "description": "an IPv4-mapped IPv6 address is invalid", + "data": "::ffff:192.168.0.1", + "valid": false + }, + { + "description": "single octet out of range in last position", + "data": "192.168.0.256", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json new file mode 100644 index 000000000..e5b46a00f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/ipv6.json @@ -0,0 +1,266 @@ +[ + { + "description": "validation of IPv6 addresses", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "ipv6" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IPv6 address", + "data": "::1", + "valid": true + }, + { + "description": "a group with 5 hex digits is invalid", + "data": "12345::", + "valid": false + }, + { + "description": "trailing 4 hex symbols is valid", + "data": "::abef", + "valid": true + }, + { + "description": "trailing 5 hex symbols is invalid", + "data": "::abcef", + "valid": false + }, + { + "description": "an IPv6 address with too many components", + "data": "1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1", + "valid": false + }, + { + "description": "an IPv6 address containing illegal characters", + "data": "::laptop", + "valid": false + }, + { + "description": "no digits is valid", + "data": "::", + "valid": true + }, + { + "description": "leading colons is valid", + "data": "::42:ff:1", + "valid": true + }, + { + "description": "trailing colons is valid", + "data": "d6::", + "valid": true + }, + { + "description": "missing leading octet is invalid", + "data": ":2:3:4:5:6:7:8", + "valid": false + }, + { + "description": "missing trailing octet is invalid", + "data": "1:2:3:4:5:6:7:", + "valid": false + }, + { + "description": "missing leading octet with omitted octets later", + "data": ":2:3:4::8", + "valid": false + }, + { + "description": "single set of double colons in the middle is valid", + "data": "1:d6::42", + "valid": true + }, + { + "description": "two sets of double colons is invalid", + "data": "1::d6::42", + "valid": false + }, + { + "description": "mixed format with the ipv4 section as decimal octets", + "data": "1::d6:192.168.0.1", + "valid": true + }, + { + "description": "mixed format with double colons between the sections", + "data": "1:2::192.168.0.1", + "valid": true + }, + { + "description": "mixed format with ipv4 section with octet out of range", + "data": "1::2:192.168.256.1", + "valid": false + }, + { + "description": "mixed format with ipv4 section with a hex octet", + "data": "1::2:192.168.ff.1", + "valid": false + }, + { + "description": "mixed format with leading double colons (ipv4-mapped ipv6 address)", + "data": "::ffff:192.168.0.1", + "valid": true + }, + { + "description": "triple colons is invalid", + "data": "1:2:3:4:5:::8", + "valid": false + }, + { + "description": "8 octets", + "data": "1:2:3:4:5:6:7:8", + "valid": true + }, + { + "description": "insufficient octets without double colons", + "data": "1:2:3:4:5:6:7", + "valid": false + }, + { + "description": "no colons is invalid", + "data": "1", + "valid": false + }, + { + "description": "ipv4 is not ipv6", + "data": "127.0.0.1", + "valid": false + }, + { + "description": "ipv4 segment must have 4 octets", + "data": "1:2:3:4:1.2.3", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " ::1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "::1 ", + "valid": false + }, + { + "description": "netmask is not a part of ipv6 address", + "data": "fe80::/64", + "valid": false + }, + { + "description": "zone id is not a part of ipv6 address", + "data": "fe80::a%eth1", + "valid": false + }, + { + "description": "a long valid ipv6", + "data": "1000:1000:1000:1000:1000:1000:255.255.255.255", + "valid": true + }, + { + "description": "a long invalid ipv6, below length limit, first", + "data": "100:100:100:100:100:100:255.255.255.255.255", + "valid": false + }, + { + "description": "a long invalid ipv6, below length limit, second", + "data": "100:100:100:100:100:100:100:255.255.255.255", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1:2:3:4:5:6:7:৪", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in the IPv4 portion", + "data": "1:2::192.16৪.0.1", + "valid": false + }, + { + "description": "a bracketed address is invalid", + "data": "[::1]", + "valid": false + }, + { + "description": "a leading zero in the last IPv4 octet is invalid", + "data": "::ffff:192.168.0.01", + "valid": false + }, + { + "description": "uppercase hex digits are valid", + "data": "2001:DB8::1", + "valid": true + }, + { + "description": "leading zeros in every group are valid", + "data": "2001:0db8:0000:0000:0000:0000:0000:0001", + "valid": true + }, + { + "description": "two groups before '::' and four groups after is valid", + "data": "1:2::3:4:5:6", + "valid": true + }, + { + "description": "eight groups alongside '::' is invalid", + "data": "1:2:3:4:5:6:7:8::", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "::1\n", + "valid": false + }, + { + "description": "an out-of-range first octet in the embedded IPv4 is invalid", + "data": "::ffff:256.1.1.1", + "valid": false + }, + { + "description": "one group before '::' and five groups after is valid", + "data": "1::2:3:4:5:6", + "valid": true + }, + { + "description": "'::' followed by six groups is valid", + "data": "::1:2:3:4:5:6", + "valid": true + }, + { + "description": "leading '::' before five groups and an embedded IPv4 is valid", + "data": "::1:2:3:4:5:1.2.3.4", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json new file mode 100644 index 000000000..d31cdbbea --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri-reference.json @@ -0,0 +1,116 @@ +[ + { + "description": "validation of IRI References", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid protocol-relative IRI Reference", + "data": "//ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid relative IRI Reference", + "data": "/âππ", + "valid": true + }, + { + "description": "an invalid IRI Reference", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "a valid IRI Reference", + "data": "âππ", + "valid": true + }, + { + "description": "a valid IRI fragment", + "data": "#ƒrägmênt", + "valid": true + }, + { + "description": "an invalid IRI fragment", + "data": "#ƒräg\\mênt", + "valid": false + }, + { + "description": "an embedded IPv4 with a leading zero is invalid", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a valid protocol-relative IRI Reference with a compressed IPv6 host", + "data": "//[2001:db8::1]/p", + "valid": true + }, + { + "description": "a valid relative IRI Reference with a supplementary-plane character", + "data": "/âππ/𐌀", + "valid": true + }, + { + "description": "a query-only IRI Reference with a supplementary private-use char", + "data": "?q=󰀀", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI Reference is invalid", + "data": "/âππ\n", + "valid": false + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "/%", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json new file mode 100644 index 000000000..db27c829a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/iri.json @@ -0,0 +1,141 @@ +[ + { + "description": "validation of IRIs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "iri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI with anchor tag", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid IRI with anchor tag and parentheses", + "data": "http://ƒøø.com/blah_(wîkïpédiå)_blah#ßité-1", + "valid": true + }, + { + "description": "a valid IRI with URL-encoded stuff", + "data": "http://ƒøø.ßår/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://ƒøø.ßår/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://ƒøø.ßår/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://ƒøø.ßår/%", + "valid": false + }, + { + "description": "a valid IRI with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid IRI based on IPv6", + "data": "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "valid": true + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + }, + { + "description": "an invalid relative IRI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid IRI", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "an invalid IRI though valid IRI reference", + "data": "âππ", + "valid": false + }, + { + "description": "an IPv6 host whose embedded IPv4 has a leading zero is invalid", + "data": "http://[::ffff:192.168.0.01]", + "valid": false + }, + { + "description": "a valid IRI with a compressed IPv6 host", + "data": "http://[2001:db8::1]", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI is invalid", + "data": "http://ƒøø.ßår/\n", + "valid": false + }, + { + "description": "a valid IRI with an IPv4 host", + "data": "http://192.168.0.1/p", + "valid": true + }, + { + "description": "a valid IRI with no authority and a rootless path", + "data": "urn:example:resource", + "valid": true + }, + { + "description": "a valid IRI with no authority and an absolute path", + "data": "file:/etc/hosts", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane character in the path", + "data": "http://ƒøø.ßår/𐌀", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane private-use char in query", + "data": "http://ƒøø.ßår/?q=󰀀", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json new file mode 100644 index 000000000..d5b9a8bb1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/json-pointer.json @@ -0,0 +1,211 @@ +[ + { + "description": "validation of JSON-pointers (JSON String Representation)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid JSON-pointer", + "data": "/foo/bar~0/baz~1/%a", + "valid": true + }, + { + "description": "not a valid JSON-pointer (~ not escaped)", + "data": "/foo/bar~", + "valid": false + }, + { + "description": "valid JSON-pointer with empty segment", + "data": "/foo//bar", + "valid": true + }, + { + "description": "valid JSON-pointer with the last empty segment", + "data": "/foo/bar/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #1", + "data": "", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #2", + "data": "/foo", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #3", + "data": "/foo/0", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #4", + "data": "/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #5", + "data": "/a~1b", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #6", + "data": "/c%d", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #7", + "data": "/e^f", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #8", + "data": "/g|h", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #9", + "data": "/i\\j", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #10", + "data": "/k\"l", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #11", + "data": "/ ", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #12", + "data": "/m~0n", + "valid": true + }, + { + "description": "valid JSON-pointer used adding to the last array position", + "data": "/foo/-", + "valid": true + }, + { + "description": "valid JSON-pointer (- used as object member name)", + "data": "/foo/-/bar", + "valid": true + }, + { + "description": "valid JSON-pointer (multiple escaped characters)", + "data": "/~1~0~0~1~1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #1", + "data": "/~1.1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #2", + "data": "/~0.1", + "valid": true + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #1", + "data": "#", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #2", + "data": "#/", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #3", + "data": "#a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #1", + "data": "/~0~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #2", + "data": "/~0/~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #1", + "data": "/~2", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #2", + "data": "/~-1", + "valid": false + }, + { + "description": "not a valid JSON-pointer (multiple characters not escaped)", + "data": "/~~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #1", + "data": "a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #2", + "data": "0", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #3", + "data": "a/a", + "valid": false + }, + { + "description": "valid JSON-pointer (Unicode characters allowed by RFC 6901)", + "data": "/foo/bar/😎", + "valid": true + }, + { + "description": "valid JSON-pointer (control characters allowed after JSON unescaping)", + "data": "/foo\u0000bar\n\tbaz", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json new file mode 100644 index 000000000..dc3e72937 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/regex.json @@ -0,0 +1,103 @@ +[ + { + "description": "validation of regular expressions", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "regex" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid regular expression", + "data": "([abc])+\\s+$", + "valid": true + }, + { + "description": "a regular expression with unclosed parens is invalid", + "data": "^(abc]", + "valid": false + }, + { + "description": "a quantifier with nothing to repeat is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns a Quantifier follows something (Term :: Atom Quantifier), so a pattern cannot begin with one", + "data": "*a", + "valid": false + }, + { + "description": "a character class range in reverse order is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns-static-semantics-early-errors it is a Syntax Error if the first ClassAtom of a range has a greater CharacterValue than the second", + "data": "[z-a]", + "valid": false + }, + { + "description": "a trailing backslash is invalid", + "data": "\\", + "valid": false + }, + { + "description": "an unmatched closing parenthesis is invalid", + "data": "a)", + "valid": false + }, + { + "description": "an unterminated character class is invalid", + "data": "[abc", + "valid": false + }, + { + "description": "an empty regular expression is valid", + "data": "", + "valid": true + }, + { + "description": "alternation is valid", + "data": "a|b", + "valid": true + }, + { + "description": "a bounded quantifier is valid", + "data": "a{2,3}", + "valid": true + }, + { + "description": "a negated range character class is valid", + "data": "[^a-z]", + "valid": true + }, + { + "description": "a lazy quantifier is valid", + "data": "a+?", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json new file mode 100644 index 000000000..413c03fd8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/relative-json-pointer.json @@ -0,0 +1,141 @@ +[ + { + "description": "validation of Relative JSON Pointers (RJP)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "relative-json-pointer" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid upwards RJP", + "data": "1", + "valid": true + }, + { + "description": "a valid downwards RJP", + "data": "0/foo/bar", + "valid": true + }, + { + "description": "a valid up and then down RJP, with array index", + "data": "2/0/baz/1/zip", + "valid": true + }, + { + "description": "a valid RJP taking the member or index name", + "data": "0#", + "valid": true + }, + { + "description": "an invalid RJP that is a valid JSON Pointer", + "data": "/foo/bar", + "valid": false + }, + { + "description": "a json-pointer part that does not start with a slash is invalid", + "data": "1foo", + "valid": false + }, + { + "description": "negative prefix", + "data": "-1/foo/bar", + "valid": false + }, + { + "description": "explicit positive prefix", + "data": "+1/foo/bar", + "valid": false + }, + { + "description": "non-ASCII digit in the prefix is not allowed", + "data": "١/foo", + "valid": false + }, + { + "description": "## is not a valid json-pointer", + "data": "0##", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus json-pointer", + "data": "01/a", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus octothorpe", + "data": "01#", + "valid": false + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "multi-digit integer prefix", + "data": "120/foo/bar", + "valid": true + }, + { + "description": "multi-digit prefix with a zero followed by another digit", + "data": "100", + "valid": true + }, + { + "description": "tilde not followed by 0 or 1 in the json-pointer part", + "data": "0/~2", + "valid": false + }, + { + "description": "unescaped tilde at the end of the json-pointer part", + "data": "0/foo/bar~", + "valid": false + }, + { + "description": "octothorpe followed by a json-pointer", + "data": "1#/foo/bar", + "valid": false + }, + { + "description": "empty reference tokens in the json-pointer part", + "data": "0//", + "valid": true + }, + { + "description": "trailing newline after the non-negative integer", + "data": "1\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json new file mode 100644 index 000000000..2f4b6353f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/time.json @@ -0,0 +1,303 @@ +[ + { + "description": "validation of time strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "time" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid time string", + "data": "08:30:06Z", + "valid": true + }, + { + "description": "invalid time string with extra leading zeros", + "data": "008:030:006Z", + "valid": false + }, + { + "description": "invalid time string with no leading zero for single digit", + "data": "8:3:6Z", + "valid": false + }, + { + "description": "hour, minute, second must be two digits", + "data": "8:0030:6Z", + "valid": false + }, + { + "description": "a valid time string with leap second, Zulu", + "data": "23:59:60Z", + "valid": true + }, + { + "description": "invalid leap second, Zulu (wrong hour)", + "data": "22:59:60Z", + "valid": false + }, + { + "description": "invalid leap second, Zulu (wrong minute)", + "data": "23:58:60Z", + "valid": false + }, + { + "description": "valid leap second, zero time-offset", + "data": "23:59:60+00:00", + "valid": true + }, + { + "description": "invalid leap second, zero time-offset (wrong hour)", + "data": "22:59:60+00:00", + "valid": false + }, + { + "description": "invalid leap second, zero time-offset (wrong minute)", + "data": "23:58:60+00:00", + "valid": false + }, + { + "description": "valid leap second, positive time-offset", + "data": "01:29:60+01:30", + "valid": true + }, + { + "description": "valid leap second, large positive time-offset", + "data": "23:29:60+23:30", + "valid": true + }, + { + "description": "invalid leap second, positive time-offset (wrong hour)", + "data": "23:59:60+01:00", + "valid": false + }, + { + "description": "invalid leap second, positive time-offset (wrong minute)", + "data": "23:59:60+00:30", + "valid": false + }, + { + "description": "valid leap second, negative time-offset", + "data": "15:59:60-08:00", + "valid": true + }, + { + "description": "valid leap second, large negative time-offset", + "data": "00:29:60-23:30", + "valid": true + }, + { + "description": "invalid leap second, negative time-offset (wrong hour)", + "data": "23:59:60-01:00", + "valid": false + }, + { + "description": "invalid leap second, negative time-offset (wrong minute)", + "data": "23:59:60-00:30", + "valid": false + }, + { + "description": "a valid time string with second fraction", + "data": "23:20:50.52Z", + "valid": true + }, + { + "description": "a valid time string with precise second fraction", + "data": "08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid time string with plus offset", + "data": "08:30:06+00:20", + "valid": true + }, + { + "description": "a valid time string with minus offset", + "data": "08:30:06-08:00", + "valid": true + }, + { + "description": "time with unknown local offset is valid", + "comment": "RFC 3339 section 4.3 (unknown local offset)", + "data": "12:34:56-00:00", + "valid": true + }, + { + "description": "hour, minute in time-offset must be two digits", + "data": "08:30:06-8:000", + "valid": false + }, + { + "description": "a valid time string with case-insensitive Z", + "data": "08:30:06z", + "valid": true + }, + { + "description": "an invalid time string with invalid hour", + "data": "24:00:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid minute", + "data": "00:60:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid second", + "data": "00:00:61Z", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset hour", + "data": "01:02:03+24:00", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset minute", + "data": "01:02:03+00:60", + "valid": false + }, + { + "description": "an invalid time string with invalid time with both Z and numoffset", + "data": "01:02:03Z+00:30", + "valid": false + }, + { + "description": "an invalid offset indicator", + "data": "08:30:06 PST", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "01:01:01,1111", + "valid": false + }, + { + "description": "no time offset", + "data": "12:00:00", + "valid": false + }, + { + "description": "no time offset with second fraction", + "data": "12:00:00.52", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২:00:00Z", + "valid": false + }, + { + "description": "offset not starting with plus or minus", + "data": "08:30:06#00:20", + "valid": false + }, + { + "description": "contains letters", + "data": "ab:cd:ef", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "a numeric time-offset requires the colon separator", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+0130", + "valid": false + }, + { + "description": "a numeric time-offset requires the minutes component", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+01", + "valid": false + }, + { + "description": "a trailing newline after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits a line feed", + "data": "08:30:06Z\n", + "valid": false + }, + { + "description": "hour 24 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-hour = 2DIGIT ; 00-23, and section 5.7 only allows 00 to 23", + "data": "24:59:00+01:00", + "valid": false + }, + { + "description": "minute 60 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-minute = 2DIGIT ; 00-59", + "data": "23:60:00+00:01", + "valid": false + }, + { + "description": "a valid time string with a 15 digit second fraction", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT, with no upper bound on the number of digits", + "data": "00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "08:30:06.Z", + "valid": false + }, + { + "description": "partial-time requires the seconds component", + "comment": "RFC 3339 section 5.6: partial-time = time-hour \":\" time-minute \":\" time-second [time-secfrac]", + "data": "12:00Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "08:30:06,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": " 08:30:06Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": "08:30:06Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json new file mode 100644 index 000000000..7fc35f536 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/unknown.json @@ -0,0 +1,46 @@ +[ + { + "description": "unknown format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "unknown" + }, + "tests": [ + { + "description": "unknown formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "unknown formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "unknown formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "unknown formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "unknown formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "unknown formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "unknown formats ignore strings", + "data": "string", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json new file mode 100644 index 000000000..08c655e00 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-reference.json @@ -0,0 +1,182 @@ +[ + { + "description": "validation of URI References", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-reference" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URI", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid relative URI Reference", + "data": "/abc", + "valid": true + }, + { + "description": "an invalid URI Reference", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "a valid URI Reference", + "data": "abc", + "valid": true + }, + { + "description": "a valid URI fragment", + "data": "#fragment", + "valid": true + }, + { + "description": "an invalid URI fragment", + "data": "#frag\\ment", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "an empty URI reference", + "comment": "RFC 3986, Section 4.2: relative-part includes path-empty, which is zero characters.", + "data": "", + "valid": true + }, + { + "description": "a query-only reference", + "comment": "RFC 3986, Section 4.2: relative-ref = relative-part [ \"?\" query ] [ \"#\" fragment ], so a query with an empty path is a reference.", + "data": "?query=1", + "valid": true + }, + { + "description": "a network-path reference with an empty authority", + "comment": "RFC 3986, Section 3.2.2: host = IP-literal / IPv4address / reg-name, and reg-name may be empty.", + "data": "//", + "valid": true + }, + { + "description": "an incomplete percent-encoding", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%zz", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%", + "valid": false + }, + { + "description": "a double quote in a path", + "comment": "RFC 3986, Appendix A: pchar = unreserved / pct-encoded / sub-delims / \":\" / \"@\", and the double quote is in none of them.", + "data": "/a\"b", + "valid": false + }, + { + "description": "square brackets outside an authority", + "comment": "RFC 3986, Section 3.2.2: square brackets are reserved for an IP-literal inside the authority. path-absolute is built from pchar, which excludes them.", + "data": "/[::1]", + "valid": false + }, + { + "description": "a non-numeric port in a network-path reference", + "comment": "RFC 3986, Section 3.2.3: port = *DIGIT. reg-name admits no colon, so the text after the colon can only be a port.", + "data": "//example.com:abc/p", + "valid": false + }, + { + "description": "more than one at-sign in the authority", + "comment": "RFC 3986, Section 3.2: authority = [ userinfo \"@\" ] host [ \":\" port ]. Neither userinfo nor host admits a literal at-sign.", + "data": "//a@b@example.com/", + "valid": false + }, + { + "description": "a leading zero in the IPv4 part of an IPv6 literal", + "comment": "RFC 3986, Section 3.2.2: inside an IP-literal the dotted quad must be an IPv4address, and dec-octet has no leading-zero alternative. There is no reg-name fallback inside brackets.", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a colon in the first segment of a relative-path reference", + "comment": "RFC 3986, Section 4.2: relative-part uses path-noscheme, whose first segment is segment-nz-nc - a non-zero-length segment without any colon. A scheme must start with ALPHA, so 1:b cannot be read as a URI either.", + "data": "1:b", + "valid": false + }, + { + "description": "a colon in the first segment preceded by a dot-segment", + "comment": "RFC 3986, Section 4.2: \"Such a segment must be preceded by a dot-segment (e.g., \\\"./this:that\\\") to make a relative-path reference.\"", + "data": "./this:that", + "valid": true + }, + { + "description": "a trailing line feed", + "comment": "RFC 3986, Appendix A: no production admits %x0A.", + "data": "/p\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json new file mode 100644 index 000000000..58d28f9de --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri-template.json @@ -0,0 +1,251 @@ +[ + { + "description": "format: uri-template", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri-template" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an invalid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": false + }, + { + "description": "a valid uri-template without variables", + "data": "http://example.com/dictionary", + "valid": true + }, + { + "description": "a valid relative uri-template", + "data": "dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an empty expression is invalid", + "data": "{}", + "valid": false + }, + { + "description": "an empty varspec inside the variable list is invalid", + "data": "{a,,b}", + "valid": false + }, + { + "description": "a zero prefix max-length is invalid", + "data": "{v:0}", + "valid": false + }, + { + "description": "a five-digit prefix max-length is invalid", + "data": "{v:10000}", + "valid": false + }, + { + "description": "an empty string is a valid uri-template", + "data": "", + "valid": true + }, + { + "description": "an unmatched closing brace is invalid", + "data": "foo}bar", + "valid": false + }, + { + "description": "a percent-encoded triplet in a literal is valid", + "data": "a%41b", + "valid": true + }, + { + "description": "a literal with a lone percent sign is invalid", + "data": "a%", + "valid": false + }, + { + "description": "a literal with an incomplete percent-encoded triplet is invalid", + "data": "a%4", + "valid": false + }, + { + "description": "a literal with non-hex percent encoding is invalid", + "data": "a%GG", + "valid": false + }, + { + "description": "a supplementary plane character in a literal is valid", + "data": "a\ud83d\ude00b", + "valid": true + }, + { + "description": "a private-use character in a literal is valid", + "data": "a\ue000b", + "valid": true + }, + { + "description": "a supplementary private-use character in a literal is valid", + "data": "a\udb80\udc00b", + "valid": true + }, + { + "description": "reserved expansion operator", + "data": "{+var}", + "valid": true + }, + { + "description": "fragment expansion operator", + "data": "{#var}", + "valid": true + }, + { + "description": "label expansion operator", + "data": "{.var}", + "valid": true + }, + { + "description": "path segment expansion operator", + "data": "{/var}", + "valid": true + }, + { + "description": "path style parameter expansion operator", + "data": "{;var}", + "valid": true + }, + { + "description": "form style query expansion operator", + "data": "{?var}", + "valid": true + }, + { + "description": "form style query continuation operator", + "data": "{&var}", + "valid": true + }, + { + "description": "the explode modifier is valid", + "data": "{var*}", + "valid": true + }, + { + "description": "combined prefix and explode modifiers are invalid", + "data": "{var:1*}", + "valid": false + }, + { + "description": "a variable list with more than one varspec is valid", + "data": "{x,y}", + "valid": true + }, + { + "description": "a dotted variable name is valid", + "data": "{a.b}", + "valid": true + }, + { + "description": "a trailing dot in a variable name is invalid", + "data": "{a.}", + "valid": false + }, + { + "description": "a double dot in a variable name is invalid", + "data": "{a..b}", + "valid": false + }, + { + "description": "a variable name may start with a percent-encoded triplet", + "data": "{%41}", + "valid": true + }, + { + "description": "a variable name with a lone percent sign is invalid", + "data": "{%}", + "valid": false + }, + { + "description": "a variable name with an incomplete percent-encoded triplet is invalid", + "data": "{%4}", + "valid": false + }, + { + "description": "a variable name with non-hex percent encoding is invalid", + "data": "{%GG}", + "valid": false + }, + { + "description": "a four-digit prefix max-length is valid", + "data": "{v:1000}", + "valid": true + }, + { + "description": "a leading zero in the prefix max-length is invalid", + "data": "{v:01}", + "valid": false + }, + { + "description": "a space in a literal is invalid", + "data": "a b", + "valid": false + }, + { + "description": "a delete character in a literal is invalid", + "data": "a\u007fb", + "valid": false + }, + { + "description": "a trailing comma in the variable list is invalid", + "data": "{a,}", + "valid": false + }, + { + "description": "a default value in a varspec is invalid", + "data": "{var=def}", + "valid": false + }, + { + "description": "an operator in place of the first varspec is invalid", + "data": "{,+var}", + "valid": false + }, + { + "description": "an apostrophe in a literal is valid", + "data": "a'b", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json new file mode 100644 index 000000000..e3736ccaf --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uri.json @@ -0,0 +1,253 @@ +[ + { + "description": "validation of URIs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uri" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URL with anchor tag", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid URL with anchor tag and parentheses", + "data": "http://foo.com/blah_(wikipedia)_blah#cite-1", + "valid": true + }, + { + "description": "a valid URL with URL-encoded stuff", + "data": "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "a valid puny-coded URL ", + "data": "http://xn--nw2a.xn--j6w193g/", + "valid": true + }, + { + "description": "a valid URL with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid URL based on IPv4", + "data": "http://223.255.255.254", + "valid": true + }, + { + "description": "a valid URL with ftp scheme", + "data": "ftp://ftp.is.co.za/rfc/rfc1808.txt", + "valid": true + }, + { + "description": "a valid URL for a simple text file", + "data": "http://www.ietf.org/rfc/rfc2396.txt", + "valid": true + }, + { + "description": "a valid URL", + "data": "ldap://[2001:db8::7]/c=GB?objectClass?one", + "valid": true + }, + { + "description": "a valid mailto URI", + "data": "mailto:John.Doe@example.com", + "valid": true + }, + { + "description": "a valid newsgroup URI", + "data": "news:comp.infosystems.www.servers.unix", + "valid": true + }, + { + "description": "a valid tel URI", + "data": "tel:+1-816-555-1212", + "valid": true + }, + { + "description": "a valid URN", + "data": "urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "valid": true + }, + { + "description": "an invalid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": false + }, + { + "description": "an invalid relative URI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid URI", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "an invalid URI though valid URI reference", + "data": "abc", + "valid": false + }, + { + "description": "an invalid URI with spaces", + "data": "http:// shouldfail.com", + "valid": false + }, + { + "description": "an invalid URI with spaces and missing scheme", + "data": ":// should fail", + "valid": false + }, + { + "description": "an invalid URI with comma in scheme", + "data": "bar,baz:foo", + "valid": false + }, + { + "description": "invalid userinfo", + "data": "https://[@example.org/test.txt", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "https://example.org/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "invalid \" character", + "data": "https://example.org/foobar\".txt", + "valid": false + }, + { + "description": "invalid <> characters", + "data": "https://example.org/foobar<>.txt", + "valid": false + }, + { + "description": "invalid {} characters", + "data": "https://example.org/foobar{}.txt", + "valid": false + }, + { + "description": "invalid ^ character", + "data": "https://example.org/foobar^.txt", + "valid": false + }, + { + "description": "invalid ` character", + "data": "https://example.org/foobar`.txt", + "valid": false + }, + { + "description": "invalid SPACE character", + "data": "https://example.org/foo bar.txt", + "valid": false + }, + { + "description": "invalid | character", + "data": "https://example.org/foobar|.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://example.com/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://example.com/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://example.com/%", + "valid": false + }, + { + "description": "scheme must start with a letter", + "data": "1http://example.com", + "valid": false + }, + { + "description": "invalid character in scheme", + "data": "ht_tp://example.com", + "valid": false + }, + { + "description": "non-numeric port is invalid", + "data": "http://example.com:abc/path", + "valid": false + }, + { + "description": "leading zero in an embedded IPv4 address is invalid", + "data": "http://[::ffff:01.2.3.4]", + "valid": false + }, + { + "description": "square brackets are not allowed in a path segment", + "data": "http:/[::1]", + "valid": false + }, + { + "description": "a trailing newline after a valid URI is invalid", + "data": "http://foo.bar/\n", + "valid": false + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json new file mode 100644 index 000000000..4679e996a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/format/uuid.json @@ -0,0 +1,156 @@ +[ + { + "description": "uuid format", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "format": "uuid" + }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "all upper-case", + "data": "2EB8AA08-AA98-11EA-B4AA-73B441D16380", + "valid": true + }, + { + "description": "all lower-case", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": true + }, + { + "description": "mixed case", + "data": "2eb8aa08-AA98-11ea-B4Aa-73B441D16380", + "valid": true + }, + { + "description": "all zeroes is valid", + "data": "00000000-0000-0000-0000-000000000000", + "valid": true + }, + { + "description": "wrong length", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1638", + "valid": false + }, + { + "description": "missing section", + "data": "2eb8aa08-aa98-11ea-73b441d16380", + "valid": false + }, + { + "description": "bad characters (not hex)", + "data": "2eb8aa08-aa98-11ea-b4ga-73b441d16380", + "valid": false + }, + { + "description": "no dashes", + "data": "2eb8aa08aa9811eab4aa73b441d16380", + "valid": false + }, + { + "description": "too few dashes", + "data": "2eb8aa08aa98-11ea-b4aa73b441d16380", + "valid": false + }, + { + "description": "too many dashes", + "data": "2eb8-aa08-aa98-11ea-b4aa73b44-1d16380", + "valid": false + }, + { + "description": "dashes in the wrong spot", + "data": "2eb8aa08aa9811eab4aa73b441d16380----", + "valid": false + }, + { + "description": "shifted dashes", + "data": "2eb8aa0-8aa98-11e-ab4aa7-3b441d16380", + "valid": false + }, + { + "description": "valid version 4", + "data": "98d80576-482e-427f-8434-7f86890ab222", + "valid": true + }, + { + "description": "valid version 5", + "data": "99c17cbb-656f-564a-940f-1a4568f03487", + "valid": true + }, + { + "description": "hypothetical version 6", + "data": "99c17cbb-656f-664a-940f-1a4568f03487", + "valid": true + }, + { + "description": "hypothetical version 15", + "data": "99c17cbb-656f-f64a-940f-1a4568f03487", + "valid": true + }, + { + "description": "URN prefixed UUID is invalid", + "data": "urn:uuid:2eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": false + }, + { + "description": "a UUID wrapped in curly braces is invalid", + "data": "{2eb8aa08-aa98-11ea-b4aa-73b441d16380}", + "valid": false + }, + { + "description": "trailing hyphen after a complete UUID is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380-", + "valid": false + }, + { + "description": "non-ASCII digit '২' (a Bengali 2) is invalid", + "data": "২eb8aa08-aa98-11ea-b4aa-73b441d16380", + "valid": false + }, + { + "description": "an underscore in place of a hex digit is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d1_380", + "valid": false + }, + { + "description": "a trailing newline after a complete UUID is invalid", + "data": "2eb8aa08-aa98-11ea-b4aa-73b441d16380\n", + "valid": false + }, + { + "description": "a variant nibble not defined by RFC 4122 is valid", + "data": "2eb8aa08-aa98-11ea-f4aa-73b441d16380", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json new file mode 100644 index 000000000..0b7df4e80 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/id.json @@ -0,0 +1,53 @@ +[ + { + "description": "$id inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an $id buried in the enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "id_in_enum": { + "enum": [ + { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + } + ] + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "string" + }, + "zzz_id_in_const": { + "const": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/id_in_enum" }, + { "$ref": "https://localhost:1234/draft2020-12/id/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$id": "https://localhost:1234/draft2020-12/id/my_identifier.json", + "type": "null" + }, + "valid": true + }, + { + "description": "match $ref to $id", + "data": "a string to match #/$defs/id_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to $id", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json new file mode 100644 index 000000000..676e6b53a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/no-schema.json @@ -0,0 +1,26 @@ +[ + { + "description": "validation without $schema", + "comment": "minLength is the same across all drafts", + "schema": { + "minLength": 2 + }, + "tests": [ + { + "description": "a 3-character string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a 1-character string is not valid", + "data": "a", + "valid": false + }, + { + "description": "a non-string is valid", + "data": 5, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json new file mode 100644 index 000000000..d2efb3ef6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/non-bmp-regex.json @@ -0,0 +1,86 @@ +[ + { + "description": "Proper UTF-16 surrogate pair handling: pattern", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^🐲*$" + }, + "tests": [ + { + "description": "matches empty", + "data": "", + "valid": true + }, + { + "description": "matches single", + "data": "🐲", + "valid": true + }, + { + "description": "matches two", + "data": "🐲🐲", + "valid": true + }, + { + "description": "doesn't match one", + "data": "🐉", + "valid": false + }, + { + "description": "doesn't match two", + "data": "🐉🐉", + "valid": false + }, + { + "description": "doesn't match one ASCII", + "data": "D", + "valid": false + }, + { + "description": "doesn't match two ASCII", + "data": "DD", + "valid": false + } + ] + }, + { + "description": "Proper UTF-16 surrogate pair handling: patternProperties", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^🐲*$": { + "type": "integer" + } + } + }, + "tests": [ + { + "description": "matches empty", + "data": { "": 1 }, + "valid": true + }, + { + "description": "matches single", + "data": { "🐲": 1 }, + "valid": true + }, + { + "description": "matches two", + "data": { "🐲🐲": 1 }, + "valid": true + }, + { + "description": "doesn't match one", + "data": { "🐲": "hello" }, + "valid": false + }, + { + "description": "doesn't match two", + "data": { "🐲🐲": "hello" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json new file mode 100644 index 000000000..ab1510dac --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/refOfUnknownKeyword.json @@ -0,0 +1,112 @@ +[ + { + "description": "reference of a root arbitrary keyword ", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknown-keyword": {"type": "integer"}, + "properties": { + "bar": {"$ref": "#/unknown-keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference of a root arbitrary keyword with encoded ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unknown/keyword": {"type": "integer"}, + "properties": { + "bar": {"$ref": "#/unknown~1keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference of an arbitrary keyword of a sub-schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"unknown-keyword": {"type": "integer"}}, + "bar": {"$ref": "#/properties/foo/unknown-keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "reference internals of known non-applicator", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "examples": [ + { "type": "string" } + ], + "$ref": "#/examples/0" + }, + "tests": [ + { + "description": "match", + "data": "a string", + "valid": true + }, + { + "description": "mismatch", + "data": 42, + "valid": false + } + ] + }, + { + "description": "reference of an arbitrary keyword of a sub-schema with encoded ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"unknown/keyword": {"type": "integer"}}, + "bar": {"$ref": "#/properties/foo/unknown~1keyword"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json new file mode 100644 index 000000000..28b0c4ce9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/optional/unknownKeyword.json @@ -0,0 +1,57 @@ +[ + { + "description": "$id inside an unknown keyword is not a real identifier", + "comment": "the implementation must not be confused by an $id in locations we do not know how to parse", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "id_in_unknown0": { + "not": { + "array_of_schemas": [ + { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "null" + } + ] + } + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "string" + }, + "id_in_unknown1": { + "not": { + "object_of_schemas": { + "foo": { + "$id": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json", + "type": "integer" + } + } + } + } + }, + "anyOf": [ + { "$ref": "#/$defs/id_in_unknown0" }, + { "$ref": "#/$defs/id_in_unknown1" }, + { "$ref": "https://localhost:1234/draft2020-12/unknownKeyword/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "type matches second anyOf, which has a real schema in it", + "data": "a string", + "valid": true + }, + { + "description": "type matches non-schema in first anyOf", + "data": null, + "valid": false + }, + { + "description": "type matches non-schema in third anyOf", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json new file mode 100644 index 000000000..ed41f79a7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/pattern.json @@ -0,0 +1,90 @@ +[ + { + "description": "pattern validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "^a*$" + }, + "tests": [ + { + "description": "a matching pattern is valid", + "data": "aaa", + "valid": true + }, + { + "description": "a non-matching pattern is invalid", + "data": "abc", + "valid": false + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "pattern is not anchored", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "pattern": "a+" + }, + "tests": [ + { + "description": "matches a substring", + "data": "xxaayy", + "valid": true + } + ] + }, + { + "description": "pattern with Unicode property escape requires unicode mode", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + "pattern": "^\\p{Letter}+$" + }, + "tests": [ + { + "description": "ASCII letters match", + "data": "Hello", + "valid": true + }, + { + "description": "Non-ASCII letters match", + "data": "π", + "valid": true + }, + { + "description": "Digits do not match", + "data": "123", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json new file mode 100644 index 000000000..5a0ce941b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/patternProperties.json @@ -0,0 +1,211 @@ +[ + { + "description": + "patternProperties validates properties matching a regex", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "f.*o": {"type": "integer"} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "multiple valid matches is valid", + "data": {"foo": 1, "foooooo" : 2}, + "valid": true + }, + { + "description": "a single invalid match is invalid", + "data": {"foo": "bar", "fooooo": 2}, + "valid": false + }, + { + "description": "multiple invalid matches is invalid", + "data": {"foo": "bar", "foooooo" : "baz"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["foo"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "multiple simultaneous patternProperties are validated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "a*": {"type": "integer"}, + "aaa*": {"maximum": 20} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"a": 21}, + "valid": true + }, + { + "description": "a simultaneous match is valid", + "data": {"aaaa": 18}, + "valid": true + }, + { + "description": "multiple matches is valid", + "data": {"a": 21, "aaaa": 18}, + "valid": true + }, + { + "description": "an invalid due to one is invalid", + "data": {"a": "bar"}, + "valid": false + }, + { + "description": "an invalid due to the other is invalid", + "data": {"aaaa": 31}, + "valid": false + }, + { + "description": "an invalid due to both is invalid", + "data": {"aaa": "foo", "aaaa": 31}, + "valid": false + } + ] + }, + { + "description": "regexes are not anchored by default and are case sensitive", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "[0-9]{2,}": { "type": "boolean" }, + "X_": { "type": "string" } + } + }, + "tests": [ + { + "description": "non recognized members are ignored", + "data": { "answer 1": "42" }, + "valid": true + }, + { + "description": "recognized members are accounted for", + "data": { "a31b": null }, + "valid": false + }, + { + "description": "regexes are case sensitive", + "data": { "a_x_3": 3 }, + "valid": true + }, + { + "description": "regexes are case sensitive, 2", + "data": { "a_X_3": 3 }, + "valid": false + } + ] + }, + { + "description": "patternProperties with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "f.*": true, + "b.*": false + } + }, + "tests": [ + { + "description": "object with property matching schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property matching schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "object with a property matching both true and false is invalid", + "data": {"foobar":1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "patternProperties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^.*bar$": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foobar": null}, + "valid": true + } + ] + }, + { + "description": "patternProperties with Unicode property escape", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "patternProperties": { + "^\\p{Letter}+$": { + "type": "number" + } + } + }, + "tests": [ + { + "description": "Unicode letter property name matches", + "data": { + "π": 1 + }, + "valid": true + }, + { + "description": "Unicode letter property name matches but value type is incorrect", + "data": { + "π": "1" + }, + "valid": false + }, + { + "description": "Non-letter property name does not match pattern", + "data": { + "123": 1 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json new file mode 100644 index 000000000..0adfc069e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/prefixItems.json @@ -0,0 +1,104 @@ +[ + { + "description": "a schema given for prefixItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "integer"}, + {"type": "string"} + ] + }, + "tests": [ + { + "description": "correct types", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "wrong types", + "data": [ "foo", 1 ], + "valid": false + }, + { + "description": "incomplete array of items", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with additional items", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "1": "valid", + "length": 2 + }, + "valid": true + } + ] + }, + { + "description": "prefixItems with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [true, false] + }, + "tests": [ + { + "description": "array with one item is valid", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with two items is invalid", + "data": [ 1, "foo" ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "additional items are allowed by default", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "integer"}] + }, + "tests": [ + { + "description": "only the first item is validated", + "data": [1, "foo", false], + "valid": true + } + ] + }, + { + "description": "prefixItems with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { + "type": "null" + } + ] + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json new file mode 100644 index 000000000..eb66fa8bd --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/properties.json @@ -0,0 +1,242 @@ +[ + { + "description": "object properties validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "both properties present and valid is valid", + "data": {"foo": 1, "bar": "baz"}, + "valid": true + }, + { + "description": "one property invalid is invalid", + "data": {"foo": 1, "bar": {}}, + "valid": false + }, + { + "description": "both properties invalid is invalid", + "data": {"foo": [], "bar": {}}, + "valid": false + }, + { + "description": "doesn't invalidate other properties", + "data": {"quux": []}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": + "properties, patternProperties, additionalProperties interaction", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "array", "maxItems": 3}, + "bar": {"type": "array"} + }, + "patternProperties": {"f.o": {"minItems": 2}}, + "additionalProperties": {"type": "integer"} + }, + "tests": [ + { + "description": "property validates property", + "data": {"foo": [1, 2]}, + "valid": true + }, + { + "description": "property invalidates property", + "data": {"foo": [1, 2, 3, 4]}, + "valid": false + }, + { + "description": "patternProperty invalidates property", + "data": {"foo": []}, + "valid": false + }, + { + "description": "patternProperty validates nonproperty", + "data": {"fxo": [1, 2]}, + "valid": true + }, + { + "description": "patternProperty invalidates nonproperty", + "data": {"fxo": []}, + "valid": false + }, + { + "description": "additionalProperty ignores property", + "data": {"bar": []}, + "valid": true + }, + { + "description": "additionalProperty validates others", + "data": {"quux": 3}, + "valid": true + }, + { + "description": "additionalProperty invalidates others", + "data": {"quux": "foo"}, + "valid": false + } + ] + }, + { + "description": "properties with boolean schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "no property present is valid", + "data": {}, + "valid": true + }, + { + "description": "only 'true' property present is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "only 'false' property present is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "both properties present is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + } + ] + }, + { + "description": "properties with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo\nbar": {"type": "number"}, + "foo\"bar": {"type": "number"}, + "foo\\bar": {"type": "number"}, + "foo\rbar": {"type": "number"}, + "foo\tbar": {"type": "number"}, + "foo\fbar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with all numbers is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1", + "foo\\bar": "1", + "foo\rbar": "1", + "foo\tbar": "1", + "foo\fbar": "1" + }, + "valid": false + } + ] + }, + { + "description": "properties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "__proto__": {"type": "number"}, + "toString": { + "properties": { "length": { "type": "string" } } + }, + "constructor": {"type": "number"} + } + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": true + }, + { + "description": "__proto__ not valid", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString not valid", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor not valid", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present and valid", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json new file mode 100644 index 000000000..84ca2b84e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/propertyNames.json @@ -0,0 +1,178 @@ +[ + { + "description": "propertyNames validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"maxLength": 3} + }, + "tests": [ + { + "description": "all property names valid", + "data": { + "f": {}, + "foo": {} + }, + "valid": true + }, + { + "description": "some property names invalid", + "data": { + "foo": {}, + "foobar": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [1, 2, 3, 4], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "propertyNames validation with pattern", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": { "pattern": "^a+$" } + }, + "tests": [ + { + "description": "matching property names valid", + "data": { + "a": {}, + "aa": {}, + "aaa": {} + }, + "valid": true + }, + { + "description": "non-matching property name is invalid", + "data": { + "aaA": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": true + }, + "tests": [ + { + "description": "object with any properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": false + }, + "tests": [ + { + "description": "object with any properties is invalid", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with const", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"const": "foo"} + }, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with enum", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"enum": ["foo", "bar"]} + }, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property foo and bar is valid", + "data": {"foo": 1, "bar": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"baz": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json new file mode 100644 index 000000000..9ca82eeb0 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/ref.json @@ -0,0 +1,1085 @@ +[ + { + "description": "root pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "match", + "data": {"foo": false}, + "valid": true + }, + { + "description": "recursive match", + "data": {"foo": {"foo": false}}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": false}, + "valid": false + }, + { + "description": "recursive mismatch", + "data": {"foo": {"bar": false}}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + {"type": "integer"}, + {"$ref": "#/prefixItems/0"} + ] + }, + "tests": [ + { + "description": "match array", + "data": [1, 2], + "valid": true + }, + { + "description": "mismatch array", + "data": [1, "foo"], + "valid": false + } + ] + }, + { + "description": "escaped pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "tilde~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"} + }, + "properties": { + "tilde": {"$ref": "#/$defs/tilde~0field"}, + "slash": {"$ref": "#/$defs/slash~1field"}, + "percent": {"$ref": "#/$defs/percent%25field"} + } + }, + "tests": [ + { + "description": "slash invalid", + "data": {"slash": "aoeu"}, + "valid": false + }, + { + "description": "tilde invalid", + "data": {"tilde": "aoeu"}, + "valid": false + }, + { + "description": "percent invalid", + "data": {"percent": "aoeu"}, + "valid": false + }, + { + "description": "slash valid", + "data": {"slash": 123}, + "valid": true + }, + { + "description": "tilde valid", + "data": {"tilde": 123}, + "valid": true + }, + { + "description": "percent valid", + "data": {"percent": 123}, + "valid": true + } + ] + }, + { + "description": "nested refs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "a": {"type": "integer"}, + "b": {"$ref": "#/$defs/a"}, + "c": {"$ref": "#/$defs/b"} + }, + "$ref": "#/$defs/c" + }, + "tests": [ + { + "description": "nested ref valid", + "data": 5, + "valid": true + }, + { + "description": "nested ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref applies alongside sibling keywords", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/$defs/reffed", + "maxItems": 2 + } + } + }, + "tests": [ + { + "description": "ref valid, maxItems valid", + "data": { "foo": [] }, + "valid": true + }, + { + "description": "ref valid, maxItems invalid", + "data": { "foo": [1, 2, 3] }, + "valid": false + }, + { + "description": "ref invalid", + "data": { "foo": "string" }, + "valid": false + } + ] + }, + { + "description": "remote ref, containing refs itself", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "https://json-schema.org/draft/2020-12/schema" + }, + "tests": [ + { + "description": "remote ref valid", + "data": {"minLength": 1}, + "valid": true + }, + { + "description": "remote ref invalid", + "data": {"minLength": -1}, + "valid": false + } + ] + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "$ref": {"type": "string"} + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "property named $ref, containing an actual $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "$ref": {"$ref": "#/$defs/is-string"} + }, + "$defs": { + "is-string": { + "type": "string" + } + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "$ref to boolean schema true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bool", + "$defs": { + "bool": true + } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to boolean schema false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bool", + "$defs": { + "bool": false + } + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "Recursive references between schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/tree", + "description": "tree of nodes", + "type": "object", + "properties": { + "meta": {"type": "string"}, + "nodes": { + "type": "array", + "items": {"$ref": "node"} + } + }, + "required": ["meta", "nodes"], + "$defs": { + "node": { + "$id": "http://localhost:1234/draft2020-12/node", + "description": "node", + "type": "object", + "properties": { + "value": {"type": "number"}, + "subtree": {"$ref": "tree"} + }, + "required": ["value"] + } + } + }, + "tests": [ + { + "description": "valid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 1.1}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": true + }, + { + "description": "invalid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": "string is invalid"}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": false + } + ] + }, + { + "description": "refs with quote", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo\"bar": {"$ref": "#/$defs/foo%22bar"} + }, + "$defs": { + "foo\"bar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with numbers is valid", + "data": { + "foo\"bar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "ref creates new scope when adjacent to keywords", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "A": { + "unevaluatedProperties": false + } + }, + "properties": { + "prop1": { + "type": "string" + } + }, + "$ref": "#/$defs/A" + }, + "tests": [ + { + "description": "referenced subschema doesn't see annotations from properties", + "data": { + "prop1": "match" + }, + "valid": false + } + ] + }, + { + "description": "naive replacement of $ref with its destination is not correct", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "a_string": { "type": "string" } + }, + "enum": [ + { "$ref": "#/$defs/a_string" } + ] + }, + "tests": [ + { + "description": "do not evaluate the $ref inside the enum, matching any string", + "data": "this is a string", + "valid": false + }, + { + "description": "do not evaluate the $ref inside the enum, definition exact match", + "data": { "type": "string" }, + "valid": false + }, + { + "description": "match the enum exactly", + "data": { "$ref": "#/$defs/a_string" }, + "valid": true + } + ] + }, + { + "description": "refs with relative uris and defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/schema-relative-uri-defs1.json", + "properties": { + "foo": { + "$id": "schema-relative-uri-defs2.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" + } + }, + "$ref": "schema-relative-uri-defs2.json" + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "relative refs with absolute uris and defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/schema-refs-absolute-uris-defs1.json", + "properties": { + "foo": { + "$id": "http://example.com/schema-refs-absolute-uris-defs2.json", + "$defs": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "$ref": "#/$defs/inner" + } + }, + "$ref": "schema-refs-absolute-uris-defs2.json" + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "$id must be resolved against nearest parent, not just immediate parent", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/a.json", + "$defs": { + "x": { + "$id": "http://example.com/b/c.json", + "not": { + "$defs": { + "y": { + "$id": "d.json", + "type": "number" + } + } + } + } + }, + "allOf": [ + { + "$ref": "http://example.com/b/d.json" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $ref", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id1/base.json", + "$ref": "int.json", + "$defs": { + "bigint": { + "$comment": "canonical uri: https://example.com/ref-and-id1/int.json", + "$id": "int.json", + "maximum": 10 + }, + "smallint": { + "$comment": "canonical uri: https://example.com/ref-and-id1-int.json", + "$id": "/draft2020-12/ref-and-id1-int.json", + "maximum": 2 + } + } + }, + "tests": [ + { + "description": "data is valid against first definition", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against first definition", + "data": 50, + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $anchor and $ref", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id2/base.json", + "$ref": "#bigint", + "$defs": { + "bigint": { + "$comment": "canonical uri: /ref-and-id2/base.json#/$defs/bigint; another valid uri for this location: /ref-and-id2/base.json#bigint", + "$anchor": "bigint", + "maximum": 10 + }, + "smallint": { + "$comment": "canonical uri: https://example.com/ref-and-id2#/$defs/smallint; another valid uri for this location: https://example.com/ref-and-id2/#bigint", + "$id": "https://example.com/draft2020-12/ref-and-id2/", + "$anchor": "bigint", + "maximum": 2 + } + } + }, + "tests": [ + { + "description": "data is valid against first definition", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against first definition", + "data": 50, + "valid": false + } + ] + }, + { + "description": "order of evaluation: $id and $ref on nested schema", + "schema": { + "$comment": "$id must be evaluated before $ref to get the proper $ref destination", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/draft2020-12/ref-and-id3/base.json", + "$ref": "nested/foo.json", + "$defs": { + "foo": { + "$comment": "canonical uri: https://example.com/draft2020-12/ref-and-id3/nested/foo.json", + "$id": "nested/foo.json", + "$ref": "./bar.json" + }, + "bar": { + "$comment": "canonical uri: https://example.com/draft2020-12/ref-and-id3/nested/bar.json", + "$id": "nested/bar.json", + "type": "number" + } + } + }, + "tests": [ + { + "description": "data is valid against nested sibling", + "data": 5, + "valid": true + }, + { + "description": "data is invalid against nested sibling", + "data": "a", + "valid": false + } + ] + }, + { + "description": "simple URN base URI with $ref via the URN", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed", + "minimum": 30, + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed"} + } + }, + "tests": [ + { + "description": "valid under the URN IDed schema", + "data": {"foo": 37}, + "valid": true + }, + { + "description": "invalid under the URN IDed schema", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "simple URN base URI with JSON pointer", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-00ff-ff00-4321feebdaed", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with NSS", + "schema": { + "$comment": "RFC 8141 §2.2", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:1/406/47452/2", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with r-component", + "schema": { + "$comment": "RFC 8141 §2.3.1", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:foo-bar-baz-qux?+CCResolve:cc=uk", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with q-component", + "schema": { + "$comment": "RFC 8141 §2.3.2", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:example:weather?=op=map&lat=39.56&lon=-104.85&datetime=1969-07-21T02:56:15Z", + "properties": { + "foo": {"$ref": "#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and JSON pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed#/$defs/bar"} + }, + "$defs": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and anchor ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed#something"} + }, + "$defs": { + "bar": { + "$anchor": "something", + "type": "string" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN ref with nested pointer ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "urn:uuid:deadbeef-4321-ffff-ffff-1234feebdaed", + "$defs": { + "foo": { + "$id": "urn:uuid:deadbeef-4321-ffff-ffff-1234feebdaed", + "$defs": {"bar": {"type": "string"}}, + "$ref": "#/$defs/bar" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": "bar", + "valid": true + }, + { + "description": "a non-string is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "ref to if", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/if", + "if": { + "$id": "http://example.com/ref/if", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to then", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/then", + "then": { + "$id": "http://example.com/ref/then", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://example.com/ref/else", + "else": { + "$id": "http://example.com/ref/else", + "type": "integer" + } + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref with absolute-path-reference", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://example.com/ref/absref.json", + "$defs": { + "a": { + "$id": "http://example.com/ref/absref/foobar.json", + "type": "number" + }, + "b": { + "$id": "http://example.com/absref/foobar.json", + "type": "string" + } + }, + "$ref": "/absref/foobar.json" + }, + "tests": [ + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "an integer is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - *nix", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "file:///folder/file.json", + "$defs": { + "foo": { + "type": "number" + } + }, + "$ref": "#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - windows", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "file:///c:/folder/file.json", + "$defs": { + "foo": { + "type": "number" + } + }, + "$ref": "#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "empty tokens in $ref json-pointer", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "": { + "$defs": { + "": { "type": "number" } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs//$defs/" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json new file mode 100644 index 000000000..6083aabb1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/refRemote.json @@ -0,0 +1,342 @@ +[ + { + "description": "remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/integer.json" + }, + "tests": [ + { + "description": "remote ref valid", + "data": 1, + "valid": true + }, + { + "description": "remote ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "fragment within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/subSchemas.json#/$defs/integer" + }, + "tests": [ + { + "description": "remote fragment valid", + "data": 1, + "valid": true + }, + { + "description": "remote fragment invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "anchor within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/locationIndependentIdentifier.json#foo" + }, + "tests": [ + { + "description": "remote anchor valid", + "data": 1, + "valid": true + }, + { + "description": "remote anchor invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref within remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/subSchemas.json#/$defs/refToInteger" + }, + "tests": [ + { + "description": "ref within ref valid", + "data": 1, + "valid": true + }, + { + "description": "ref within ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "base URI change", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/", + "items": { + "$id": "baseUriChange/", + "items": {"$ref": "folderInteger.json"} + } + }, + "tests": [ + { + "description": "base URI change ref valid", + "data": [[1]], + "valid": true + }, + { + "description": "base URI change ref invalid", + "data": [["a"]], + "valid": false + } + ] + }, + { + "description": "base URI change - change folder", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/scope_change_defs1.json", + "type" : "object", + "properties": {"list": {"$ref": "baseUriChangeFolder/"}}, + "$defs": { + "baz": { + "$id": "baseUriChangeFolder/", + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "base URI change - change folder in subschema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/scope_change_defs2.json", + "type" : "object", + "properties": {"list": {"$ref": "baseUriChangeFolderInSubschema/#/$defs/bar"}}, + "$defs": { + "baz": { + "$id": "baseUriChangeFolderInSubschema/", + "$defs": { + "bar": { + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "root ref in remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/object", + "type": "object", + "properties": { + "name": {"$ref": "name-defs.json#/$defs/orNull"} + } + }, + "tests": [ + { + "description": "string is valid", + "data": { + "name": "foo" + }, + "valid": true + }, + { + "description": "null is valid", + "data": { + "name": null + }, + "valid": true + }, + { + "description": "object is invalid", + "data": { + "name": { + "name": null + } + }, + "valid": false + } + ] + }, + { + "description": "remote ref with ref to defs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/schema-remote-ref-ref-defs1.json", + "$ref": "ref-and-defs.json" + }, + "tests": [ + { + "description": "invalid", + "data": { + "bar": 1 + }, + "valid": false + }, + { + "description": "valid", + "data": { + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "Location-independent identifier in remote ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/locationIndependentIdentifier.json#/$defs/refToInteger" + }, + "tests": [ + { + "description": "integer is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "retrieved nested refs resolve relative to their URI not $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "http://localhost:1234/draft2020-12/some-id", + "properties": { + "name": {"$ref": "nested/foo-ref-string.json"} + } + }, + "tests": [ + { + "description": "number is invalid", + "data": { + "name": {"foo": 1} + }, + "valid": false + }, + { + "description": "string is valid", + "data": { + "name": {"foo": "a"} + }, + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with different $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/different-id-ref-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with different URN $id", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/urn-ref-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "remote HTTP ref with nested absolute ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/nested-absolute-ref-to-string.json" + }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to $ref finds detached $anchor", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "http://localhost:1234/draft2020-12/detached-ref.json#/$defs/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json new file mode 100644 index 000000000..17db8939e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/required.json @@ -0,0 +1,169 @@ +[ + { + "description": "required validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {}, + "bar": {} + }, + "required": ["foo"] + }, + "tests": [ + { + "description": "present required property is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "non-present required property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description":"ignores boolean", + "data": true, + "valid": true + + } + ] + }, + { + "description": "required default validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + } + }, + "tests": [ + { + "description": "not required by default", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with empty array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": {} + }, + "required": [] + }, + "tests": [ + { + "description": "property not required", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with escaped characters", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": [ + "foo\nbar", + "foo\"bar", + "foo\\bar", + "foo\rbar", + "foo\tbar", + "foo\fbar" + ] + }, + "tests": [ + { + "description": "object with all properties present is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with some properties missing is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "required properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "required": ["__proto__", "toString", "constructor"] + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": false + }, + { + "description": "__proto__ present", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString present", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor present", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json new file mode 100644 index 000000000..2123c408d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/type.json @@ -0,0 +1,501 @@ +[ + { + "description": "integer type matches integers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "integer" + }, + "tests": [ + { + "description": "an integer is an integer", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is an integer", + "data": 1.0, + "valid": true + }, + { + "description": "a float is not an integer", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an integer", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not an integer, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not an integer", + "data": {}, + "valid": false + }, + { + "description": "an array is not an integer", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an integer", + "data": true, + "valid": false + }, + { + "description": "null is not an integer", + "data": null, + "valid": false + } + ] + }, + { + "description": "number type matches numbers", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "number" + }, + "tests": [ + { + "description": "an integer is a number", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is a number (and an integer)", + "data": 1.0, + "valid": true + }, + { + "description": "a float is a number", + "data": 1.1, + "valid": true + }, + { + "description": "a string is not a number", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not a number, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not a number", + "data": {}, + "valid": false + }, + { + "description": "an array is not a number", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a number", + "data": true, + "valid": false + }, + { + "description": "null is not a number", + "data": null, + "valid": false + } + ] + }, + { + "description": "string type matches strings", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string" + }, + "tests": [ + { + "description": "1 is not a string", + "data": 1, + "valid": false + }, + { + "description": "a float is not a string", + "data": 1.1, + "valid": false + }, + { + "description": "a string is a string", + "data": "foo", + "valid": true + }, + { + "description": "a string is still a string, even if it looks like a number", + "data": "1", + "valid": true + }, + { + "description": "an empty string is still a string", + "data": "", + "valid": true + }, + { + "description": "an object is not a string", + "data": {}, + "valid": false + }, + { + "description": "an array is not a string", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a string", + "data": true, + "valid": false + }, + { + "description": "null is not a string", + "data": null, + "valid": false + } + ] + }, + { + "description": "object type matches objects", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object" + }, + "tests": [ + { + "description": "an integer is not an object", + "data": 1, + "valid": false + }, + { + "description": "a float is not an object", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an object", + "data": "foo", + "valid": false + }, + { + "description": "an object is an object", + "data": {}, + "valid": true + }, + { + "description": "an array is not an object", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an object", + "data": true, + "valid": false + }, + { + "description": "null is not an object", + "data": null, + "valid": false + } + ] + }, + { + "description": "array type matches arrays", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array" + }, + "tests": [ + { + "description": "an integer is not an array", + "data": 1, + "valid": false + }, + { + "description": "a float is not an array", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an array", + "data": "foo", + "valid": false + }, + { + "description": "an object is not an array", + "data": {}, + "valid": false + }, + { + "description": "an array is an array", + "data": [], + "valid": true + }, + { + "description": "a boolean is not an array", + "data": true, + "valid": false + }, + { + "description": "null is not an array", + "data": null, + "valid": false + } + ] + }, + { + "description": "boolean type matches booleans", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "boolean" + }, + "tests": [ + { + "description": "an integer is not a boolean", + "data": 1, + "valid": false + }, + { + "description": "zero is not a boolean", + "data": 0, + "valid": false + }, + { + "description": "a float is not a boolean", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not a boolean", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not a boolean", + "data": "", + "valid": false + }, + { + "description": "an object is not a boolean", + "data": {}, + "valid": false + }, + { + "description": "an array is not a boolean", + "data": [], + "valid": false + }, + { + "description": "true is a boolean", + "data": true, + "valid": true + }, + { + "description": "false is a boolean", + "data": false, + "valid": true + }, + { + "description": "null is not a boolean", + "data": null, + "valid": false + } + ] + }, + { + "description": "null type matches only the null object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "null" + }, + "tests": [ + { + "description": "an integer is not null", + "data": 1, + "valid": false + }, + { + "description": "a float is not null", + "data": 1.1, + "valid": false + }, + { + "description": "zero is not null", + "data": 0, + "valid": false + }, + { + "description": "a string is not null", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not null", + "data": "", + "valid": false + }, + { + "description": "an object is not null", + "data": {}, + "valid": false + }, + { + "description": "an array is not null", + "data": [], + "valid": false + }, + { + "description": "true is not null", + "data": true, + "valid": false + }, + { + "description": "false is not null", + "data": false, + "valid": false + }, + { + "description": "null is null", + "data": null, + "valid": true + } + ] + }, + { + "description": "multiple types can be specified in an array", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["integer", "string"] + }, + "tests": [ + { + "description": "an integer is valid", + "data": 1, + "valid": true + }, + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a float is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "an object is invalid", + "data": {}, + "valid": false + }, + { + "description": "an array is invalid", + "data": [], + "valid": false + }, + { + "description": "a boolean is invalid", + "data": true, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type as array with one item", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["string"] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "type: array or object", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["array", "object"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type: array, object or null", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": ["array", "object", "null"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json new file mode 100644 index 000000000..d8a7c1b5b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedItems.json @@ -0,0 +1,854 @@ +[ + { + "description": "unevaluatedItems true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": true + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems as schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": { "type": "string" } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with valid unevaluated items", + "data": ["foo"], + "valid": true + }, + { + "description": "with invalid unevaluated items", + "data": [42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with uniform items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": { "type": "string" }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "unevaluatedItems doesn't apply", + "data": ["foo", "bar"], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with tuple", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with items and prefixItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "items": true, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "unevaluatedItems doesn't apply", + "data": ["foo", 42], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "items": {"type": "number"}, + "unevaluatedItems": {"type": "string"} + }, + "tests": [ + { + "description": "valid under items", + "comment": "no elements are considered by unevaluatedItems", + "data": [5, 6, 7, 8], + "valid": true + }, + { + "description": "invalid under items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested tuple", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "type": "string" } + ], + "allOf": [ + { + "prefixItems": [ + true, + { "type": "number" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", 42], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", 42, true], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": {"type": "boolean"}, + "anyOf": [ + { "items": {"type": "string"} }, + true + ] + }, + "tests": [ + { + "description": "with only (valid) additional items", + "data": [true, false], + "valid": true + }, + { + "description": "with no additional items", + "data": ["yes", "no"], + "valid": true + }, + { + "description": "with invalid additional item", + "data": ["yes", false], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with nested prefixItems and items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ + { "type": "string" } + ], + "items": true + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no additional items", + "data": ["foo"], + "valid": true + }, + { + "description": "with additional items", + "data": ["foo", 42, true], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with nested unevaluatedItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ + { "type": "string" } + ] + }, + { "unevaluatedItems": true } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no additional items", + "data": ["foo"], + "valid": true + }, + { + "description": "with additional items", + "data": ["foo", 42, true], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "anyOf": [ + { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + { + "prefixItems": [ + true, + true, + { "const": "baz" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "when one schema matches and has no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "when one schema matches and has unevaluated items", + "data": ["foo", "bar", 42], + "valid": false + }, + { + "description": "when two schemas match and has no unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "when two schemas match and has unevaluated items", + "data": ["foo", "bar", "baz", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "oneOf": [ + { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + { + "prefixItems": [ + true, + { "const": "baz" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "not": { + "not": { + "prefixItems": [ + true, + { "const": "bar" } + ] + } + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with unevaluated items", + "data": ["foo", "bar"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with if/then/else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { "const": "foo" } + ], + "if": { + "prefixItems": [ + true, + { "const": "bar" } + ] + }, + "then": { + "prefixItems": [ + true, + true, + { "const": "then" } + ] + }, + "else": { + "prefixItems": [ + true, + true, + true, + { "const": "else" } + ] + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "when if matches and it has no unevaluated items", + "data": ["foo", "bar", "then"], + "valid": true + }, + { + "description": "when if matches and it has unevaluated items", + "data": ["foo", "bar", "then", "else"], + "valid": false + }, + { + "description": "when if doesn't match and it has no unevaluated items", + "data": ["foo", 42, 42, "else"], + "valid": true + }, + { + "description": "when if doesn't match and it has unevaluated items", + "data": ["foo", 42, 42, "else", 42], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [true], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": [], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bar", + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false, + "$defs": { + "bar": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems before $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false, + "prefixItems": [ + { "type": "string" } + ], + "$ref": "#/$defs/bar", + "$defs": { + "bar": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/unevaluated-items-with-dynamic-ref/derived", + + "$ref": "./baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "prefixItems": [ + true, + { "type": "string" } + ] + }, + "baseSchema": { + "$id": "./baseSchema", + + "$comment": "unevaluatedItems comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedItems": false, + "type": "array", + "prefixItems": [ + { "type": "string" } + ], + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated items", + "data": ["foo", "bar"], + "valid": true + }, + { + "description": "with unevaluated items", + "data": ["foo", "bar", "baz"], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems can't see inside cousins", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "prefixItems": [ true ] + }, + { "unevaluatedItems": false } + ] + }, + "tests": [ + { + "description": "always fails", + "data": [ 1 ], + "valid": false + } + ] + }, + { + "description": "item is evaluated in an uncle schema to unevaluatedItems", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "prefixItems": [ + { "type": "string" } + ], + "unevaluatedItems": false + } + }, + "anyOf": [ + { + "properties": { + "foo": { + "prefixItems": [ + true, + { "type": "string" } + ] + } + } + } + ] + }, + "tests": [ + { + "description": "no extra items", + "data": { + "foo": [ + "test" + ] + }, + "valid": true + }, + { + "description": "uncle keyword evaluation is not significant", + "data": { + "foo": [ + "test", + "test" + ] + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedItems depends on adjacent contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [true], + "contains": {"type": "string"}, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "second item is evaluated by contains", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "contains fails, second item is not evaluated", + "data": [ 1, 2 ], + "valid": false + }, + { + "description": "contains passes, second item is not evaluated", + "data": [ 1, 2, "foo" ], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems depends on multiple nested contains", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { "contains": { "multipleOf": 2 } }, + { "contains": { "multipleOf": 3 } } + ], + "unevaluatedItems": { "multipleOf": 5 } + }, + "tests": [ + { + "description": "5 not evaluated, passes unevaluatedItems", + "data": [ 2, 3, 4, 5, 6 ], + "valid": true + }, + { + "description": "7 not evaluated, fails unevaluatedItems", + "data": [ 2, 3, 4, 7, 8 ], + "valid": false + } + ] + }, + { + "description": "unevaluatedItems and contains interact to control item dependency relationship", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "contains": {"const": "a"} + }, + "then": { + "if": { + "contains": {"const": "b"} + }, + "then": { + "if": { + "contains": {"const": "c"} + } + } + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "empty array is valid", + "data": [], + "valid": true + }, + { + "description": "only a's are valid", + "data": [ "a", "a" ], + "valid": true + }, + { + "description": "a's and b's are valid", + "data": [ "a", "b", "a", "b", "a" ], + "valid": true + }, + { + "description": "a's, b's and c's are valid", + "data": [ "c", "a", "c", "c", "b", "a" ], + "valid": true + }, + { + "description": "only b's are invalid", + "data": [ "b", "b" ], + "valid": false + }, + { + "description": "only c's are invalid", + "data": [ "c", "c" ], + "valid": false + }, + { + "description": "only b's and c's are invalid", + "data": [ "c", "b", "c", "b", "c" ], + "valid": false + }, + { + "description": "only a's and c's are invalid", + "data": [ "c", "a", "c", "a", "c" ], + "valid": false + } + ] + }, + { + "description" : "unevaluatedItems with minContains = 0", + "schema" : { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "contains": {"type": "string"}, + "minContains": 0, + "unevaluatedItems": false + }, + "tests" : [ + { + "description": "empty array is valid", + "data": [], + "valid": true + }, + { + "description": "no items evaluated by contains", + "data": [0], + "valid": false + }, + { + "description": "some but not all items evaluated by contains", + "data": ["foo", 0], + "valid": false + }, + { + "description": "all items evaluated by contains", + "data": ["foo", "bar"], + "valid": true + } + ] + }, + { + "description": "non-array instances are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": false + }, + "tests": [ + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "unevaluatedItems with null instance elements", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedItems": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "unevaluatedItems can see annotations from if without then and else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "prefixItems": [{"const": "a"}] + }, + "unevaluatedItems": false + }, + "tests": [ + { + "description": "valid in case if is evaluated", + "data": [ "a" ], + "valid": true + }, + { + "description": "invalid in case if is evaluated", + "data": [ "b" ], + "valid": false + } + ] + }, + { + "description": "Evaluated items collection needs to consider instance location", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [ + { + "prefixItems": [ + true, + { "type": "string" } + ] + } + ], + "unevaluatedItems": false + }, + "tests": [ + { + "description": "with an unevaluated item that exists at another location", + "data": [ + ["foo", "bar"], + "bar" + ], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json new file mode 100644 index 000000000..1995d0ea9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/unevaluatedProperties.json @@ -0,0 +1,1681 @@ +[ + { + "description": "unevaluatedProperties true", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties schema", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": { + "type": "string", + "minLength": 3 + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with valid unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with invalid unevaluated properties", + "data": { + "foo": "fo" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": {}, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "patternProperties": { + "^foo": { "type": "string" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with adjacent bool additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "additionalProperties": true, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with adjacent non-bool additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "foo": { "type": "string" } + }, + "additionalProperties": {"type": "string"}, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with nested properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "properties": { + "bar": { "type": "string" } + } + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with nested patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "patternProperties": { + "^bar": { "type": "string" } + } + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with nested additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "additionalProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no additional properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with additional properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with nested unevaluatedProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": { + "type": "string", + "maxLength": 2 + } + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with anyOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "anyOf": [ + { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + }, + { + "properties": { + "baz": { "const": "baz" } + }, + "required": ["baz"] + }, + { + "properties": { + "quux": { "const": "quux" } + }, + "required": ["quux"] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when one matches and has no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when one matches and has unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "not-baz" + }, + "valid": false + }, + { + "description": "when two match and has no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": true + }, + { + "description": "when two match and has unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz", + "quux": "not-quux" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "oneOf": [ + { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + }, + { + "properties": { + "baz": { "const": "baz" } + }, + "required": ["baz"] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "quux": "quux" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with not", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "not": { + "not": { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "then": { + "properties": { + "bar": { "type": "string" } + }, + "required": ["bar"] + }, + "else": { + "properties": { + "baz": { "type": "string" } + }, + "required": ["baz"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar", + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": true + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else, then not defined", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "else": { + "properties": { + "baz": { "type": "string" } + }, + "required": ["baz"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": true + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with if/then/else, else not defined", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "properties": { + "foo": { "const": "then" } + }, + "required": ["foo"] + }, + "then": { + "properties": { + "bar": { "type": "string" } + }, + "required": ["bar"] + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "when if is true and has no unevaluated properties", + "data": { + "foo": "then", + "bar": "bar" + }, + "valid": true + }, + { + "description": "when if is true and has unevaluated properties", + "data": { + "foo": "then", + "bar": "bar", + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has no unevaluated properties", + "data": { + "baz": "baz" + }, + "valid": false + }, + { + "description": "when if is false and has unevaluated properties", + "data": { + "foo": "else", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with dependentSchemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "dependentSchemas": { + "foo": { + "properties": { + "bar": { "const": "bar" } + }, + "required": ["bar"] + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with boolean schemas", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [true], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/bar", + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false, + "$defs": { + "bar": { + "properties": { + "bar": { "type": "string" } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties before $ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false, + "properties": { + "foo": { "type": "string" } + }, + "$ref": "#/$defs/bar", + "$defs": { + "bar": { + "properties": { + "bar": { "type": "string" } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties with $dynamicRef", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.com/unevaluated-properties-with-dynamic-ref/derived", + + "$ref": "./baseSchema", + + "$defs": { + "derived": { + "$dynamicAnchor": "addons", + "properties": { + "bar": { "type": "string" } + } + }, + "baseSchema": { + "$id": "./baseSchema", + + "$comment": "unevaluatedProperties comes first so it's more likely to catch bugs with implementations that are sensitive to keyword ordering", + "unevaluatedProperties": false, + "properties": { + "foo": { "type": "string" } + }, + "$dynamicRef": "#addons", + + "$defs": { + "defaultAddons": { + "$comment": "Needed to satisfy the bookending requirement", + "$dynamicAnchor": "addons" + } + } + } + } + }, + "tests": [ + { + "description": "with no unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + }, + { + "description": "with unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar", + "baz": "baz" + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can't see inside cousins", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + } + }, + { + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "always fails", + "data": { + "foo": 1 + }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can't see inside cousins (reverse order)", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "unevaluatedProperties": false + }, + { + "properties": { + "foo": true + } + } + ] + }, + "tests": [ + { + "description": "always fails", + "data": { + "foo": 1 + }, + "valid": false + } + ] + }, + { + "description": "nested unevaluatedProperties, outer false, inner true, properties outside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "nested unevaluatedProperties, outer false, inner true, properties inside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": true + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": true + } + ] + }, + { + "description": "nested unevaluatedProperties, outer true, inner false, properties outside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { "type": "string" } + }, + "allOf": [ + { + "unevaluatedProperties": false + } + ], + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "nested unevaluatedProperties, outer true, inner false, properties inside", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + } + ], + "unevaluatedProperties": true + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "cousin unevaluatedProperties, true and false, true with properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": true + }, + { + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": false + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "cousin unevaluatedProperties, true and false, false with properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "unevaluatedProperties": true + }, + { + "properties": { + "foo": { "type": "string" } + }, + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "with no nested unevaluated properties", + "data": { + "foo": "foo" + }, + "valid": true + }, + { + "description": "with nested unevaluated properties", + "data": { + "foo": "foo", + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "property is evaluated in an uncle schema to unevaluatedProperties", + "comment": "see https://stackoverflow.com/questions/66936884/deeply-nested-unevaluatedproperties-and-their-expectations", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "properties": { + "bar": { + "type": "string" + } + }, + "unevaluatedProperties": false + } + }, + "anyOf": [ + { + "properties": { + "foo": { + "properties": { + "faz": { + "type": "string" + } + } + } + } + } + ] + }, + "tests": [ + { + "description": "no extra properties", + "data": { + "foo": { + "bar": "test" + } + }, + "valid": true + }, + { + "description": "uncle keyword evaluation is not significant", + "data": { + "foo": { + "bar": "test", + "faz": "test" + } + }, + "valid": false + } + ] + }, + { + "description": "in-place applicator siblings, allOf has unevaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + }, + "unevaluatedProperties": false + } + ], + "anyOf": [ + { + "properties": { + "bar": true + } + } + ] + }, + "tests": [ + { + "description": "base case: both properties present", + "data": { + "foo": 1, + "bar": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, bar is missing", + "data": { + "foo": 1 + }, + "valid": true + }, + { + "description": "in place applicator siblings, foo is missing", + "data": { + "bar": 1 + }, + "valid": false + } + ] + }, + { + "description": "in-place applicator siblings, anyOf has unevaluated", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "allOf": [ + { + "properties": { + "foo": true + } + } + ], + "anyOf": [ + { + "properties": { + "bar": true + }, + "unevaluatedProperties": false + } + ] + }, + "tests": [ + { + "description": "base case: both properties present", + "data": { + "foo": 1, + "bar": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, bar is missing", + "data": { + "foo": 1 + }, + "valid": false + }, + { + "description": "in place applicator siblings, foo is missing", + "data": { + "bar": 1 + }, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties + single cyclic ref", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "x": { "$ref": "#" } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is valid", + "data": {}, + "valid": true + }, + { + "description": "Single is valid", + "data": { "x": {} }, + "valid": true + }, + { + "description": "Unevaluated on 1st level is invalid", + "data": { "x": {}, "y": {} }, + "valid": false + }, + { + "description": "Nested is valid", + "data": { "x": { "x": {} } }, + "valid": true + }, + { + "description": "Unevaluated on 2nd level is invalid", + "data": { "x": { "x": {}, "y": {} } }, + "valid": false + }, + { + "description": "Deep nested is valid", + "data": { "x": { "x": { "x": {} } } }, + "valid": true + }, + { + "description": "Unevaluated on 3rd level is invalid", + "data": { "x": { "x": { "x": {}, "y": {} } } }, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties + ref inside allOf / oneOf", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "one": { + "properties": { "a": true } + }, + "two": { + "required": ["x"], + "properties": { "x": true } + } + }, + "allOf": [ + { "$ref": "#/$defs/one" }, + { "properties": { "b": true } }, + { + "oneOf": [ + { "$ref": "#/$defs/two" }, + { + "required": ["y"], + "properties": { "y": true } + } + ] + } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is invalid (no x or y)", + "data": {}, + "valid": false + }, + { + "description": "a and b are invalid (no x or y)", + "data": { "a": 1, "b": 1 }, + "valid": false + }, + { + "description": "x and y are invalid", + "data": { "x": 1, "y": 1 }, + "valid": false + }, + { + "description": "a and x are valid", + "data": { "a": 1, "x": 1 }, + "valid": true + }, + { + "description": "a and y are valid", + "data": { "a": 1, "y": 1 }, + "valid": true + }, + { + "description": "a and b and x are valid", + "data": { "a": 1, "b": 1, "x": 1 }, + "valid": true + }, + { + "description": "a and b and y are valid", + "data": { "a": 1, "b": 1, "y": 1 }, + "valid": true + }, + { + "description": "a and b and x and y are invalid", + "data": { "a": 1, "b": 1, "x": 1, "y": 1 }, + "valid": false + } + ] + }, + { + "description": "dynamic evaluation inside nested refs", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "one": { + "oneOf": [ + { "$ref": "#/$defs/two" }, + { "required": ["b"], "properties": { "b": true } }, + { "required": ["xx"], "patternProperties": { "x": true } }, + { "required": ["all"], "unevaluatedProperties": true } + ] + }, + "two": { + "oneOf": [ + { "required": ["c"], "properties": { "c": true } }, + { "required": ["d"], "properties": { "d": true } } + ] + } + }, + "oneOf": [ + { "$ref": "#/$defs/one" }, + { "required": ["a"], "properties": { "a": true } } + ], + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "Empty is invalid", + "data": {}, + "valid": false + }, + { + "description": "a is valid", + "data": { "a": 1 }, + "valid": true + }, + { + "description": "b is valid", + "data": { "b": 1 }, + "valid": true + }, + { + "description": "c is valid", + "data": { "c": 1 }, + "valid": true + }, + { + "description": "d is valid", + "data": { "d": 1 }, + "valid": true + }, + { + "description": "a + b is invalid", + "data": { "a": 1, "b": 1 }, + "valid": false + }, + { + "description": "a + c is invalid", + "data": { "a": 1, "c": 1 }, + "valid": false + }, + { + "description": "a + d is invalid", + "data": { "a": 1, "d": 1 }, + "valid": false + }, + { + "description": "b + c is invalid", + "data": { "b": 1, "c": 1 }, + "valid": false + }, + { + "description": "b + d is invalid", + "data": { "b": 1, "d": 1 }, + "valid": false + }, + { + "description": "c + d is invalid", + "data": { "c": 1, "d": 1 }, + "valid": false + }, + { + "description": "xx is valid", + "data": { "xx": 1 }, + "valid": true + }, + { + "description": "xx + foox is valid", + "data": { "xx": 1, "foox": 1 }, + "valid": true + }, + { + "description": "xx + foo is invalid", + "data": { "xx": 1, "foo": 1 }, + "valid": false + }, + { + "description": "xx + a is invalid", + "data": { "xx": 1, "a": 1 }, + "valid": false + }, + { + "description": "xx + b is invalid", + "data": { "xx": 1, "b": 1 }, + "valid": false + }, + { + "description": "xx + c is invalid", + "data": { "xx": 1, "c": 1 }, + "valid": false + }, + { + "description": "xx + d is invalid", + "data": { "xx": 1, "d": 1 }, + "valid": false + }, + { + "description": "all is valid", + "data": { "all": 1 }, + "valid": true + }, + { + "description": "all + foo is valid", + "data": { "all": 1, "foo": 1 }, + "valid": true + }, + { + "description": "all + a is invalid", + "data": { "all": 1, "a": 1 }, + "valid": false + } + ] + }, + { + "description": "non-object instances are valid", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties with null valued instance properties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "unevaluatedProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null valued properties", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "unevaluatedProperties not affected by propertyNames", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "propertyNames": {"maxLength": 1}, + "unevaluatedProperties": { + "type": "number" + } + }, + "tests": [ + { + "description": "allows only number properties", + "data": {"a": 1}, + "valid": true + }, + { + "description": "string property is invalid", + "data": {"a": "b"}, + "valid": false + } + ] + }, + { + "description": "unevaluatedProperties can see annotations from if without then and else", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "if": { + "patternProperties": { + "foo": { + "type": "string" + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "valid in case if is evaluated", + "data": { + "foo": "a" + }, + "valid": true + }, + { + "description": "invalid in case if is evaluated", + "data": { + "bar": "a" + }, + "valid": false + } + ] + }, + { + "description": "dependentSchemas with unevaluatedProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": {"foo2": {}}, + "dependentSchemas": { + "foo" : {}, + "foo2": { + "properties": { + "bar":{} + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "unevaluatedProperties doesn't consider dependentSchemas", + "data": {"foo": ""}, + "valid": false + }, + { + "description": "unevaluatedProperties doesn't see bar when foo2 is absent", + "data": {"bar": ""}, + "valid": false + }, + { + "description": "unevaluatedProperties sees bar when foo2 is present", + "data": { "foo2": "", "bar": ""}, + "valid": true + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location with patternProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "patternProperties": { + "^bar$": { "type": "string" } + } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with only the nested evaluated property", + "data": { + "foo": { "bar": "foo" } + }, + "valid": true + }, + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + }, + { + "description": "Evaluated properties collection needs to consider instance location with additionalProperties", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "foo": { + "additionalProperties": { "type": "string" } + } + }, + "unevaluatedProperties": false + }, + "tests": [ + { + "description": "with only the nested evaluated property", + "data": { + "foo": { "bar": "foo" } + }, + "valid": true + }, + { + "description": "with an unevaluated property that exists at another location", + "data": { + "foo": { "bar": "foo" }, + "bar": "bar" + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json new file mode 100644 index 000000000..262114529 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/uniqueItems.json @@ -0,0 +1,399 @@ +[ + { + "description": "uniqueItems validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "uniqueItems": true + }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is invalid", + "data": [1, 1], + "valid": false + }, + { + "description": "non-unique array of more than two integers is invalid", + "data": [1, 2, 1], + "valid": false + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": false + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of strings is valid", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "non-unique array of strings is invalid", + "data": ["foo", "bar", "foo"], + "valid": false + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is invalid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": false + }, + { + "description": "property order of array of objects is ignored", + "data": [{"foo": "bar", "bar": "foo"}, {"bar": "foo", "foo": "bar"}], + "valid": false + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is invalid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": false + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is invalid", + "data": [["foo"], ["foo"]], + "valid": false + }, + { + "description": "non-unique array of more than two arrays is invalid", + "data": [["foo"], ["bar"], ["foo"]], + "valid": false + }, + { + "description": "[1] and [true] are unique", + "data": [[1], [true]], + "valid": true + }, + { + "description": "[0] and [false] are unique", + "data": [[0], [false]], + "valid": true + }, + { + "description": "nested [1] and [true] are unique", + "data": [[[1], "foo"], [[true], "foo"]], + "valid": true + }, + { + "description": "nested [0] and [false] are unique", + "data": [[[0], "foo"], [[false], "foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1, "{}"], + "valid": true + }, + { + "description": "non-unique heterogeneous types are invalid", + "data": [{}, [1], true, null, {}, 1], + "valid": false + }, + { + "description": "different objects are unique", + "data": [{"a": 1, "b": 2}, {"a": 2, "b": 1}], + "valid": true + }, + { + "description": "objects are non-unique despite key order", + "data": [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + "valid": false + }, + { + "description": "{\"a\": false} and {\"a\": 0} are unique", + "data": [{"a": false}, {"a": 0}], + "valid": true + }, + { + "description": "{\"a\": true} and {\"a\": 1} are unique", + "data": [{"a": true}, {"a": 1}], + "valid": true + } + ] + }, + { + "description": "uniqueItems with an array of items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is not valid", + "data": [false, true, "foo", "foo"], + "valid": false + }, + { + "description": "non-unique array extended from [true, false] is not valid", + "data": [true, false, "foo", "foo"], + "valid": false + } + ] + }, + { + "description": "uniqueItems with an array of items and additionalItems=false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true, + "items": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + }, + { + "description": "uniqueItems=false validation", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "uniqueItems": false + }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is valid", + "data": [1, 1], + "valid": true + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": true + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": true + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": true + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is valid", + "data": [["foo"], ["foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1], + "valid": true + }, + { + "description": "non-unique heterogeneous types are valid", + "data": [{}, [1], true, null, {}, 1], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is valid", + "data": [false, true, "foo", "foo"], + "valid": true + }, + { + "description": "non-unique array extended from [true, false] is valid", + "data": [true, false, "foo", "foo"], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items and additionalItems=false", + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "prefixItems": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false, + "items": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json new file mode 100644 index 000000000..1acb96a93 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft2020-12/vocabulary.json @@ -0,0 +1,57 @@ +[ + { + "description": "schema that uses custom metaschema with with no validation vocabulary", + "schema": { + "$id": "https://schema/using/no/validation", + "$schema": "http://localhost:1234/draft2020-12/metaschema-no-validation.json", + "properties": { + "badProperty": false, + "numberProperty": { + "minimum": 10 + } + } + }, + "tests": [ + { + "description": "applicator vocabulary still works", + "data": { + "badProperty": "this property should not exist" + }, + "valid": false + }, + { + "description": "no validation: valid number", + "data": { + "numberProperty": 20 + }, + "valid": true + }, + { + "description": "no validation: invalid number, but it still validates", + "data": { + "numberProperty": 1 + }, + "valid": true + } + ] + }, + { + "description": "ignore unrecognized optional vocabulary", + "schema": { + "$schema": "http://localhost:1234/draft2020-12/metaschema-optional-vocabulary.json", + "type": "number" + }, + "tests": [ + { + "description": "string value", + "data": "foobar", + "valid": false + }, + { + "description": "number value", + "data": 20, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json new file mode 100644 index 000000000..6ef4212b1 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalItems.json @@ -0,0 +1,190 @@ +[ + { + "description": "additionalItems as schema", + "schema": { + "items": [{}], + "additionalItems": {"type": "integer"} + }, + "tests": [ + { + "description": "additional items match schema", + "data": [ null, 2, 3, 4 ], + "valid": true + }, + { + "description": "additional items do not match schema", + "data": [ null, 2, 3, "foo" ], + "valid": false + } + ] + }, + { + "description": "when items is schema, additionalItems does nothing", + "schema": { + "items": { + "type": "integer" + }, + "additionalItems": { + "type": "string" + } + }, + "tests": [ + { + "description": "valid with a array of type integers", + "data": [1,2,3], + "valid": true + }, + { + "description": "invalid with a array of mixed types", + "data": [1,"2","3"], + "valid": false + } + ] + }, + { + "description": "when items is schema, boolean additionalItems does nothing", + "schema": { + "items": {}, + "additionalItems": false + }, + "tests": [ + { + "description": "all items match schema", + "data": [ 1, 2, 3, 4, 5 ], + "valid": true + } + ] + }, + { + "description": "array of items with no additionalItems permitted", + "schema": { + "items": [{}, {}, {}], + "additionalItems": false + }, + "tests": [ + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "fewer number of items present (1)", + "data": [ 1 ], + "valid": true + }, + { + "description": "fewer number of items present (2)", + "data": [ 1, 2 ], + "valid": true + }, + { + "description": "equal number of items present", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "additional items are not permitted", + "data": [ 1, 2, 3, 4 ], + "valid": false + } + ] + }, + { + "description": "additionalItems as false without items", + "schema": {"additionalItems": false}, + "tests": [ + { + "description": + "items defaults to empty schema so everything is valid", + "data": [ 1, 2, 3, 4, 5 ], + "valid": true + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + } + ] + }, + { + "description": "additionalItems are allowed by default", + "schema": {"items": [{"type": "integer"}]}, + "tests": [ + { + "description": "only the first item is validated", + "data": [1, "foo", false], + "valid": true + } + ] + }, + { + "description": "additionalItems does not look in applicators, invalid case", + "schema": { + "allOf": [ + { "items": [ { "type": "integer" }, { "type": "string" } ] } + ], + "items": [ {"type": "integer" } ], + "additionalItems": { "type": "boolean" } + }, + "tests": [ + { + "description": "items defined in allOf are not examined", + "data": [ 1, "hello" ], + "valid": false + } + ] + }, + { + "description": "items validation adjusts the starting index for additionalItems", + "schema": { + "items": [ { "type": "string" } ], + "additionalItems": { "type": "integer" } + }, + "tests": [ + { + "description": "valid items", + "data": [ "x", 2, 3 ], + "valid": true + }, + { + "description": "wrong type of second item", + "data": [ "x", "y" ], + "valid": false + } + ] + }, + { + "description": "additionalItems with heterogeneous array", + "schema": { + "items": [{}], + "additionalItems": false + }, + "tests": [ + { + "description": "heterogeneous invalid instance", + "data": [ "foo", "bar", 37 ], + "valid": false + }, + { + "description": "valid instance", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "additionalItems with null instance elements", + "schema": { + "additionalItems": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json new file mode 100644 index 000000000..0f8e1627a --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/additionalProperties.json @@ -0,0 +1,147 @@ +[ + { + "description": + "additionalProperties being false does not allow other properties", + "schema": { + "properties": {"foo": {}, "bar": {}}, + "patternProperties": { "^v": {} }, + "additionalProperties": false + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : "boom"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobarbaz", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "patternProperties are not additional properties", + "data": {"foo":1, "vroom": 2}, + "valid": true + } + ] + }, + { + "description": "non-ASCII pattern with additionalProperties", + "schema": { + "patternProperties": {"^á": {}}, + "additionalProperties": false + }, + "tests": [ + { + "description": "matching the pattern is valid", + "data": {"ármányos": 2}, + "valid": true + }, + { + "description": "not matching the pattern is invalid", + "data": {"élmény": 2}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with schema", + "schema": { + "properties": {"foo": {}, "bar": {}}, + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "no additional properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "an additional valid property is valid", + "data": {"foo" : 1, "bar" : 2, "quux" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1, "bar" : 2, "quux" : 12}, + "valid": false + } + ] + }, + { + "description": + "additionalProperties can exist by itself", + "schema": { + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "an additional valid property is valid", + "data": {"foo" : true}, + "valid": true + }, + { + "description": "an additional invalid property is invalid", + "data": {"foo" : 1}, + "valid": false + } + ] + }, + { + "description": "additionalProperties are allowed by default", + "schema": {"properties": {"foo": {}, "bar": {}}}, + "tests": [ + { + "description": "additional properties are allowed", + "data": {"foo": 1, "bar": 2, "quux": true}, + "valid": true + } + ] + }, + { + "description": "additionalProperties does not look in applicators", + "schema": { + "allOf": [ + {"properties": {"foo": {}}} + ], + "additionalProperties": {"type": "boolean"} + }, + "tests": [ + { + "description": "properties defined in allOf are not examined", + "data": {"foo": 1, "bar": true}, + "valid": false + } + ] + }, + { + "description": "additionalProperties with null valued instance properties", + "schema": { + "additionalProperties": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json new file mode 100644 index 000000000..ec9319e14 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/allOf.json @@ -0,0 +1,294 @@ +[ + { + "description": "allOf", + "schema": { + "allOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "allOf", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "mismatch second", + "data": {"foo": "baz"}, + "valid": false + }, + { + "description": "mismatch first", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "wrong type", + "data": {"foo": "baz", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "allOf with base schema", + "schema": { + "properties": {"bar": {"type": "integer"}}, + "required": ["bar"], + "allOf" : [ + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + }, + { + "properties": { + "baz": {"type": "null"} + }, + "required": ["baz"] + } + ] + }, + "tests": [ + { + "description": "valid", + "data": {"foo": "quux", "bar": 2, "baz": null}, + "valid": true + }, + { + "description": "mismatch base schema", + "data": {"foo": "quux", "baz": null}, + "valid": false + }, + { + "description": "mismatch first allOf", + "data": {"bar": 2, "baz": null}, + "valid": false + }, + { + "description": "mismatch second allOf", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "mismatch both", + "data": {"bar": 2}, + "valid": false + } + ] + }, + { + "description": "allOf simple types", + "schema": { + "allOf": [ + {"maximum": 30}, + {"minimum": 20} + ] + }, + "tests": [ + { + "description": "valid", + "data": 25, + "valid": true + }, + { + "description": "mismatch one", + "data": 35, + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all true", + "schema": {"allOf": [true, true]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "allOf with boolean schemas, some false", + "schema": {"allOf": [true, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with boolean schemas, all false", + "schema": {"allOf": [false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with one empty schema", + "schema": { + "allOf": [ + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with two empty schemas", + "schema": { + "allOf": [ + {}, + {} + ] + }, + "tests": [ + { + "description": "any data is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "allOf with the first empty schema", + "schema": { + "allOf": [ + {}, + { "type": "number" } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "allOf with the last empty schema", + "schema": { + "allOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "nested allOf, to check validation semantics", + "schema": { + "allOf": [ + { + "allOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "allOf combined with anyOf, oneOf", + "schema": { + "allOf": [ { "multipleOf": 2 } ], + "anyOf": [ { "multipleOf": 3 } ], + "oneOf": [ { "multipleOf": 5 } ] + }, + "tests": [ + { + "description": "allOf: false, anyOf: false, oneOf: false", + "data": 1, + "valid": false + }, + { + "description": "allOf: false, anyOf: false, oneOf: true", + "data": 5, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: false", + "data": 3, + "valid": false + }, + { + "description": "allOf: false, anyOf: true, oneOf: true", + "data": 15, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: false", + "data": 2, + "valid": false + }, + { + "description": "allOf: true, anyOf: false, oneOf: true", + "data": 10, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: false", + "data": 6, + "valid": false + }, + { + "description": "allOf: true, anyOf: true, oneOf: true", + "data": 30, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json new file mode 100644 index 000000000..ab5eb386b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/anyOf.json @@ -0,0 +1,189 @@ +[ + { + "description": "anyOf", + "schema": { + "anyOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first anyOf valid", + "data": 1, + "valid": true + }, + { + "description": "second anyOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both anyOf valid", + "data": 3, + "valid": true + }, + { + "description": "neither anyOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "anyOf with base schema", + "schema": { + "type": "string", + "anyOf" : [ + { + "maxLength": 2 + }, + { + "minLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one anyOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both anyOf invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf with boolean schemas, all true", + "schema": {"anyOf": [true, true]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, some true", + "schema": {"anyOf": [true, false]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "anyOf with boolean schemas, all false", + "schema": {"anyOf": [false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "anyOf complex types", + "schema": { + "anyOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first anyOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second anyOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both anyOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": true + }, + { + "description": "neither anyOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "anyOf with one empty schema", + "schema": { + "anyOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is valid", + "data": 123, + "valid": true + } + ] + }, + { + "description": "nested anyOf, to check validation semantics", + "schema": { + "anyOf": [ + { + "anyOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json new file mode 100644 index 000000000..6d40f23f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/boolean_schema.json @@ -0,0 +1,104 @@ +[ + { + "description": "boolean schema 'true'", + "schema": true, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "boolean schema 'false'", + "schema": false, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json new file mode 100644 index 000000000..168fff4e3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/const.json @@ -0,0 +1,384 @@ +[ + { + "description": "const validation", + "schema": {"const": 2}, + "tests": [ + { + "description": "same value is valid", + "data": 2, + "valid": true + }, + { + "description": "another value is invalid", + "data": 5, + "valid": false + }, + { + "description": "another type is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "const with object", + "schema": {"const": {"foo": "bar", "baz": "bax"}}, + "tests": [ + { + "description": "same object is valid", + "data": {"foo": "bar", "baz": "bax"}, + "valid": true + }, + { + "description": "same object with different property order is valid", + "data": {"baz": "bax", "foo": "bar"}, + "valid": true + }, + { + "description": "another object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "another type is invalid", + "data": [1, 2], + "valid": false + } + ] + }, + { + "description": "const with array", + "schema": {"const": [{ "foo": "bar" }]}, + "tests": [ + { + "description": "same array is valid", + "data": [{"foo": "bar"}], + "valid": true + }, + { + "description": "another array item is invalid", + "data": [2], + "valid": false + }, + { + "description": "array with additional items is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + }, + { + "description": "const with null", + "schema": {"const": null}, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "not null is invalid", + "data": 0, + "valid": false + } + ] + }, + { + "description": "const with false does not match 0", + "schema": {"const": false}, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "const with true does not match 1", + "schema": {"const": true}, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "const with [false] does not match [0]", + "schema": {"const": [false]}, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "const with [true] does not match [1]", + "schema": {"const": [true]}, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "const with {\"a\": false} does not match {\"a\": 0}", + "schema": {"const": {"a": false}}, + "tests": [ + { + "description": "{\"a\": false} is valid", + "data": {"a": false}, + "valid": true + }, + { + "description": "{\"a\": 0} is invalid", + "data": {"a": 0}, + "valid": false + }, + { + "description": "{\"a\": 0.0} is invalid", + "data": {"a": 0.0}, + "valid": false + } + ] + }, + { + "description": "const with {\"a\": true} does not match {\"a\": 1}", + "schema": {"const": {"a": true}}, + "tests": [ + { + "description": "{\"a\": true} is valid", + "data": {"a": true}, + "valid": true + }, + { + "description": "{\"a\": 1} is invalid", + "data": {"a": 1}, + "valid": false + }, + { + "description": "{\"a\": 1.0} is invalid", + "data": {"a": 1.0}, + "valid": false + } + ] + }, + { + "description": "const with 0 does not match other zero-like types", + "schema": {"const": 0}, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + } + ] + }, + { + "description": "const with 1 does not match true", + "schema": {"const": 1}, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "const with -2.0 matches integer and float types", + "schema": {"const": -2.0}, + "tests": [ + { + "description": "integer -2 is valid", + "data": -2, + "valid": true + }, + { + "description": "integer 2 is invalid", + "data": 2, + "valid": false + }, + { + "description": "float -2.0 is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float 2.0 is invalid", + "data": 2.0, + "valid": false + }, + { + "description": "float -2.00001 is invalid", + "data": -2.00001, + "valid": false + } + ] + }, + { + "description": "float and integers are equal up to 64-bit representation limits", + "schema": {"const": 9007199254740992}, + "tests": [ + { + "description": "integer is valid", + "data": 9007199254740992, + "valid": true + }, + { + "description": "integer minus one is invalid", + "data": 9007199254740991, + "valid": false + }, + { + "description": "float is valid", + "data": 9007199254740992.0, + "valid": true + }, + { + "description": "float minus one is invalid", + "data": 9007199254740991.0, + "valid": false + } + ] + }, + { + "description": "nul characters in strings", + "schema": { "const": "hello\u0000there" }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation but different codepoint", + "schema": { + "const": "μ", + "$comment": "U+03BC" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "μ", + "comment": "U+03BC", + "valid": true + }, + { + "description": "character looks the same but uses a different codepoint", + "data": "µ", + "comment": "U+00B5", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "const": "ä", + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json new file mode 100644 index 000000000..2b1a51520 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/contains.json @@ -0,0 +1,165 @@ +[ + { + "description": "contains keyword validation", + "schema": { + "contains": {"minimum": 5} + }, + "tests": [ + { + "description": "array with item matching schema (5) is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with item matching schema (6) is valid", + "data": [3, 4, 6], + "valid": true + }, + { + "description": "array with two items matching schema (5, 6) is valid", + "data": [3, 4, 5, 6], + "valid": true + }, + { + "description": "array without items matching schema is invalid", + "data": [2, 3, 4], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "not array is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "contains keyword with const keyword", + "schema": { + "contains": { "const": 5 } + }, + "tests": [ + { + "description": "array with item 5 is valid", + "data": [3, 4, 5], + "valid": true + }, + { + "description": "array with two items 5 is valid", + "data": [3, 4, 5, 5], + "valid": true + }, + { + "description": "array without item 5 is invalid", + "data": [1, 2, 3, 4], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema true", + "schema": {"contains": true}, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains keyword with boolean schema false", + "schema": {"contains": false}, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + }, + { + "description": "non-arrays are valid", + "data": "contains does not apply to strings", + "valid": true + } + ] + }, + { + "description": "items + contains", + "schema": { + "items": { "multipleOf": 2 }, + "contains": { "multipleOf": 3 } + }, + "tests": [ + { + "description": "matches items, does not match contains", + "data": [ 2, 4, 8 ], + "valid": false + }, + { + "description": "does not match items, matches contains", + "data": [ 3, 6, 9 ], + "valid": false + }, + { + "description": "matches both items and contains", + "data": [ 6, 12 ], + "valid": true + }, + { + "description": "matches neither items nor contains", + "data": [ 1, 5 ], + "valid": false + } + ] + }, + { + "description": "contains with false if subschema", + "schema": { + "contains": { + "if": false, + "else": true + } + }, + "tests": [ + { + "description": "any non-empty array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "contains with null instance elements", + "schema": { + "contains": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null items", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json new file mode 100644 index 000000000..289a9b66c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/default.json @@ -0,0 +1,79 @@ +[ + { + "description": "invalid type for default", + "schema": { + "properties": { + "foo": { + "type": "integer", + "default": [] + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"foo": 13}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "invalid string value for default", + "schema": { + "properties": { + "bar": { + "type": "string", + "minLength": 4, + "default": "bad" + } + } + }, + "tests": [ + { + "description": "valid when property is specified", + "data": {"bar": "good"}, + "valid": true + }, + { + "description": "still valid when the invalid default is used", + "data": {}, + "valid": true + } + ] + }, + { + "description": "the default keyword does not do anything if the property is missing", + "schema": { + "type": "object", + "properties": { + "alpha": { + "type": "number", + "maximum": 3, + "default": 5 + } + } + }, + "tests": [ + { + "description": "an explicit property value is checked against maximum (passing)", + "data": { "alpha": 1 }, + "valid": true + }, + { + "description": "an explicit property value is checked against maximum (failing)", + "data": { "alpha": 5 }, + "valid": false + }, + { + "description": "missing properties are not filled in with the default", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json new file mode 100644 index 000000000..afe396e42 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/definitions.json @@ -0,0 +1,26 @@ +[ + { + "description": "validate definition against metaschema", + "schema": {"$ref": "http://json-schema.org/draft-07/schema#"}, + "tests": [ + { + "description": "valid definition schema", + "data": { + "definitions": { + "foo": {"type": "integer"} + } + }, + "valid": true + }, + { + "description": "invalid definition schema", + "data": { + "definitions": { + "foo": {"type": 1} + } + }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json new file mode 100644 index 000000000..c0bd809f6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/dependencies.json @@ -0,0 +1,286 @@ +[ + { + "description": "dependencies", + "schema": { + "dependencies": {"bar": ["foo"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependant", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "with dependency", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["bar"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "dependencies with empty array", + "schema": { + "dependencies": {"bar": []} + }, + "tests": [ + { + "description": "empty object", + "data": {}, + "valid": true + }, + { + "description": "object with one property", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "non-object is valid", + "data": 1, + "valid": true + } + ] + }, + { + "description": "multiple dependencies", + "schema": { + "dependencies": {"quux": ["foo", "bar"]} + }, + "tests": [ + { + "description": "neither", + "data": {}, + "valid": true + }, + { + "description": "nondependants", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "with dependencies", + "data": {"foo": 1, "bar": 2, "quux": 3}, + "valid": true + }, + { + "description": "missing dependency", + "data": {"foo": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing other dependency", + "data": {"bar": 1, "quux": 2}, + "valid": false + }, + { + "description": "missing both dependencies", + "data": {"quux": 1}, + "valid": false + } + ] + }, + { + "description": "multiple dependencies subschema", + "schema": { + "dependencies": { + "bar": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "integer"} + } + } + } + }, + "tests": [ + { + "description": "valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "no dependency", + "data": {"foo": "quux"}, + "valid": true + }, + { + "description": "wrong type", + "data": {"foo": "quux", "bar": 2}, + "valid": false + }, + { + "description": "wrong type other", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + }, + { + "description": "wrong type both", + "data": {"foo": "quux", "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "dependencies with boolean subschemas", + "schema": { + "dependencies": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "object with property having schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property having schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "dependencies with escaped characters", + "schema": { + "dependencies": { + "foo\nbar": ["foo\rbar"], + "foo\tbar": { + "minProperties": 4 + }, + "foo'bar": {"required": ["foo\"bar"]}, + "foo\"bar": ["foo'bar"] + } + }, + "tests": [ + { + "description": "valid object 1", + "data": { + "foo\nbar": 1, + "foo\rbar": 2 + }, + "valid": true + }, + { + "description": "valid object 2", + "data": { + "foo\tbar": 1, + "a": 2, + "b": 3, + "c": 4 + }, + "valid": true + }, + { + "description": "valid object 3", + "data": { + "foo'bar": 1, + "foo\"bar": 2 + }, + "valid": true + }, + { + "description": "invalid object 1", + "data": { + "foo\nbar": 1, + "foo": 2 + }, + "valid": false + }, + { + "description": "invalid object 2", + "data": { + "foo\tbar": 1, + "a": 2 + }, + "valid": false + }, + { + "description": "invalid object 3", + "data": { + "foo'bar": 1 + }, + "valid": false + }, + { + "description": "invalid object 4", + "data": { + "foo\"bar": 2 + }, + "valid": false + } + ] + }, + { + "description": "dependent subschema incompatible with root", + "schema": { + "properties": { + "foo": {} + }, + "dependencies": { + "foo": { + "properties": { + "bar": {} + }, + "additionalProperties": false + } + } + }, + "tests": [ + { + "description": "matches root", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "matches dependency", + "data": {"bar": 1}, + "valid": true + }, + { + "description": "matches both", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "no dependency", + "data": {"baz": 1}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json new file mode 100644 index 000000000..bf1e96fc5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/enum.json @@ -0,0 +1,341 @@ +[ + { + "description": "simple enum validation", + "schema": {"enum": [1, 2, 3]}, + "tests": [ + { + "description": "one of the enum is valid", + "data": 1, + "valid": true + }, + { + "description": "something else is invalid", + "data": 4, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum validation", + "schema": {"enum": [6, "foo", [], true, {"foo": 12}]}, + "tests": [ + { + "description": "one of the enum is valid", + "data": [], + "valid": true + }, + { + "description": "something else is invalid", + "data": null, + "valid": false + }, + { + "description": "objects are deep compared", + "data": {"foo": false}, + "valid": false + }, + { + "description": "valid object matches", + "data": {"foo": 12}, + "valid": true + }, + { + "description": "extra properties in object is invalid", + "data": {"foo": 12, "boo": 42}, + "valid": false + } + ] + }, + { + "description": "heterogeneous enum-with-null validation", + "schema": { "enum": [6, null] }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is valid", + "data": 6, + "valid": true + }, + { + "description": "something else is invalid", + "data": "test", + "valid": false + } + ] + }, + { + "description": "enums in properties", + "schema": { + "type":"object", + "properties": { + "foo": {"enum":["foo"]}, + "bar": {"enum":["bar"]} + }, + "required": ["bar"] + }, + "tests": [ + { + "description": "both properties are valid", + "data": {"foo":"foo", "bar":"bar"}, + "valid": true + }, + { + "description": "wrong foo value", + "data": {"foo":"foot", "bar":"bar"}, + "valid": false + }, + { + "description": "wrong bar value", + "data": {"foo":"foo", "bar":"bart"}, + "valid": false + }, + { + "description": "missing optional property is valid", + "data": {"bar":"bar"}, + "valid": true + }, + { + "description": "missing required property is invalid", + "data": {"foo":"foo"}, + "valid": false + }, + { + "description": "missing all properties is invalid", + "data": {}, + "valid": false + } + ] + }, + { + "description": "enum with escaped characters", + "schema": { + "enum": ["foo\nbar", "foo\rbar"] + }, + "tests": [ + { + "description": "member 1 is valid", + "data": "foo\nbar", + "valid": true + }, + { + "description": "member 2 is valid", + "data": "foo\rbar", + "valid": true + }, + { + "description": "another string is invalid", + "data": "abc", + "valid": false + } + ] + }, + { + "description": "enum with false does not match 0", + "schema": {"enum": [false]}, + "tests": [ + { + "description": "false is valid", + "data": false, + "valid": true + }, + { + "description": "integer zero is invalid", + "data": 0, + "valid": false + }, + { + "description": "float zero is invalid", + "data": 0.0, + "valid": false + } + ] + }, + { + "description": "enum with [false] does not match [0]", + "schema": {"enum": [[false]]}, + "tests": [ + { + "description": "[false] is valid", + "data": [false], + "valid": true + }, + { + "description": "[0] is invalid", + "data": [0], + "valid": false + }, + { + "description": "[0.0] is invalid", + "data": [0.0], + "valid": false + } + ] + }, + { + "description": "enum with true does not match 1", + "schema": {"enum": [true]}, + "tests": [ + { + "description": "true is valid", + "data": true, + "valid": true + }, + { + "description": "integer one is invalid", + "data": 1, + "valid": false + }, + { + "description": "float one is invalid", + "data": 1.0, + "valid": false + } + ] + }, + { + "description": "enum with [true] does not match [1]", + "schema": {"enum": [[true]]}, + "tests": [ + { + "description": "[true] is valid", + "data": [true], + "valid": true + }, + { + "description": "[1] is invalid", + "data": [1], + "valid": false + }, + { + "description": "[1.0] is invalid", + "data": [1.0], + "valid": false + } + ] + }, + { + "description": "enum with 0 does not match false", + "schema": {"enum": [0]}, + "tests": [ + { + "description": "false is invalid", + "data": false, + "valid": false + }, + { + "description": "integer zero is valid", + "data": 0, + "valid": true + }, + { + "description": "float zero is valid", + "data": 0.0, + "valid": true + } + ] + }, + { + "description": "enum with [0] does not match [false]", + "schema": {"enum": [[0]]}, + "tests": [ + { + "description": "[false] is invalid", + "data": [false], + "valid": false + }, + { + "description": "[0] is valid", + "data": [0], + "valid": true + }, + { + "description": "[0.0] is valid", + "data": [0.0], + "valid": true + } + ] + }, + { + "description": "enum with 1 does not match true", + "schema": {"enum": [1]}, + "tests": [ + { + "description": "true is invalid", + "data": true, + "valid": false + }, + { + "description": "integer one is valid", + "data": 1, + "valid": true + }, + { + "description": "float one is valid", + "data": 1.0, + "valid": true + } + ] + }, + { + "description": "enum with [1] does not match [true]", + "schema": {"enum": [[1]]}, + "tests": [ + { + "description": "[true] is invalid", + "data": [true], + "valid": false + }, + { + "description": "[1] is valid", + "data": [1], + "valid": true + }, + { + "description": "[1.0] is valid", + "data": [1.0], + "valid": true + } + ] + }, + { + "description": "nul characters in strings", + "schema": { "enum": [ "hello\u0000there" ] }, + "tests": [ + { + "description": "match string with nul", + "data": "hello\u0000there", + "valid": true + }, + { + "description": "do not match string lacking nul", + "data": "hellothere", + "valid": false + } + ] + }, + { + "description": "characters with the same visual representation, but different number of codepoints", + "schema": { + "enum": ["ä"], + "$comment": "U+00E4" + }, + "tests": [ + { + "description": "character uses the same codepoint", + "data": "ä", + "comment": "U+00E4", + "valid": true + }, + { + "description": "character looks the same but uses combining marks", + "data": "ä", + "comment": "a, U+0308", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json new file mode 100644 index 000000000..dc3cd709d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMaximum.json @@ -0,0 +1,30 @@ +[ + { + "description": "exclusiveMaximum validation", + "schema": { + "exclusiveMaximum": 3.0 + }, + "tests": [ + { + "description": "below the exclusiveMaximum is valid", + "data": 2.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 3.0, + "valid": false + }, + { + "description": "above the exclusiveMaximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json new file mode 100644 index 000000000..b38d7ecec --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/exclusiveMinimum.json @@ -0,0 +1,30 @@ +[ + { + "description": "exclusiveMinimum validation", + "schema": { + "exclusiveMinimum": 1.1 + }, + "tests": [ + { + "description": "above the exclusiveMinimum is valid", + "data": 1.2, + "valid": true + }, + { + "description": "boundary point is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "below the exclusiveMinimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json new file mode 100644 index 000000000..e2447d60f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/format.json @@ -0,0 +1,614 @@ +[ + { + "description": "email format", + "schema": { "format": "email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "idn-email format", + "schema": { "format": "idn-email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "regex format", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "ipv4 format", + "schema": { "format": "ipv4" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "ipv6 format", + "schema": { "format": "ipv6" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "idn-hostname format", + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "hostname format", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "date format", + "schema": { "format": "date" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "date-time format", + "schema": { "format": "date-time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "time format", + "schema": { "format": "time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "json-pointer format", + "schema": { "format": "json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "relative-json-pointer format", + "schema": { "format": "relative-json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "iri format", + "schema": { "format": "iri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "iri-reference format", + "schema": { "format": "iri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri format", + "schema": { "format": "uri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri-reference format", + "schema": { "format": "uri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + }, + { + "description": "uri-template format", + "schema": { "format": "uri-template" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json new file mode 100644 index 000000000..a3bab7c0b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/if-then-else.json @@ -0,0 +1,296 @@ +[ + { + "description": "ignore if without then or else", + "schema": { + "if": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone if", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone if", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore then without if", + "schema": { + "then": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone then", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone then", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "ignore else without if", + "schema": { + "else": { + "const": 0 + } + }, + "tests": [ + { + "description": "valid when valid against lone else", + "data": 0, + "valid": true + }, + { + "description": "valid when invalid against lone else", + "data": "hello", + "valid": true + } + ] + }, + { + "description": "if and then without else", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid when if test fails", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if and else without then", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid when if test passes", + "data": -1, + "valid": true + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "validate against correct branch, then vs else", + "schema": { + "if": { + "exclusiveMaximum": 0 + }, + "then": { + "minimum": -10 + }, + "else": { + "multipleOf": 2 + } + }, + "tests": [ + { + "description": "valid through then", + "data": -1, + "valid": true + }, + { + "description": "invalid through then", + "data": -100, + "valid": false + }, + { + "description": "valid through else", + "data": 4, + "valid": true + }, + { + "description": "invalid through else", + "data": 3, + "valid": false + } + ] + }, + { + "description": "non-interference across combined schemas", + "schema": { + "allOf": [ + { + "if": { + "exclusiveMaximum": 0 + } + }, + { + "then": { + "minimum": -10 + } + }, + { + "else": { + "multipleOf": 2 + } + } + ] + }, + "tests": [ + { + "description": "valid, but would have been invalid through then", + "data": -100, + "valid": true + }, + { + "description": "valid, but would have been invalid through else", + "data": 3, + "valid": true + } + ] + }, + { + "description": "if with boolean schema true", + "schema": { + "if": true, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema true in if always chooses the then path (valid)", + "data": "then", + "valid": true + }, + { + "description": "boolean schema true in if always chooses the then path (invalid)", + "data": "else", + "valid": false + } + ] + }, + { + "description": "if with boolean schema false", + "schema": { + "if": false, + "then": { "const": "then" }, + "else": { "const": "else" } + }, + "tests": [ + { + "description": "boolean schema false in if always chooses the else path (invalid)", + "data": "then", + "valid": false + }, + { + "description": "boolean schema false in if always chooses the else path (valid)", + "data": "else", + "valid": true + } + ] + }, + { + "description": "if appears at the end when serialized (keyword processing sequence)", + "schema": { + "then": { "const": "yes" }, + "else": { "const": "other" }, + "if": { "maxLength": 4 } + }, + "tests": [ + { + "description": "yes redirects to then and passes", + "data": "yes", + "valid": true + }, + { + "description": "other redirects to else and passes", + "data": "other", + "valid": true + }, + { + "description": "no redirects to then and fails", + "data": "no", + "valid": false + }, + { + "description": "invalid redirects to else and fails", + "data": "invalid", + "valid": false + } + ] + }, + { + "description": "then: false fails when condition matches", + "schema": { + "if": { "const": 1 }, + "then": false + }, + "tests": [ + { + "description": "matches if → then=false → invalid", + "data": 1, + "valid": false + }, + { + "description": "does not match if → then ignored → valid", + "data": 2, + "valid": true + } + ] + }, + { + "description": "else: false fails when condition does not match", + "schema": { + "if": { "const": 1 }, + "else": false + }, + "tests": [ + { + "description": "matches if → else ignored → valid", + "data": 1, + "valid": true + }, + { + "description": "does not match if → else executes → invalid", + "data": 2, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json new file mode 100644 index 000000000..f98c74fc6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/infinite-loop-detection.json @@ -0,0 +1,36 @@ +[ + { + "description": "evaluating the same schema location against the same data location twice is not a sign of an infinite loop", + "schema": { + "definitions": { + "int": { "type": "integer" } + }, + "allOf": [ + { + "properties": { + "foo": { + "$ref": "#/definitions/int" + } + } + }, + { + "additionalProperties": { + "$ref": "#/definitions/int" + } + } + ] + }, + "tests": [ + { + "description": "passing case", + "data": { "foo": 1 }, + "valid": true + }, + { + "description": "failing case", + "data": { "foo": "a string" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json new file mode 100644 index 000000000..7ed6781b6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/items.json @@ -0,0 +1,282 @@ +[ + { + "description": "a schema given for items", + "schema": { + "items": {"type": "integer"} + }, + "tests": [ + { + "description": "valid items", + "data": [ 1, 2, 3 ], + "valid": true + }, + { + "description": "wrong type of items", + "data": [1, "x"], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": {"foo" : "bar"}, + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "length": 1 + }, + "valid": true + } + ] + }, + { + "description": "an array of schemas for items", + "schema": { + "items": [ + {"type": "integer"}, + {"type": "string"} + ] + }, + "tests": [ + { + "description": "correct types", + "data": [ 1, "foo" ], + "valid": true + }, + { + "description": "wrong types", + "data": [ "foo", 1 ], + "valid": false + }, + { + "description": "incomplete array of items", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with additional items", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array", + "data": [ ], + "valid": true + }, + { + "description": "JavaScript pseudo-array is valid", + "data": { + "0": "invalid", + "1": "valid", + "length": 2 + }, + "valid": true + } + ] + }, + { + "description": "items with boolean schema (true)", + "schema": {"items": true}, + "tests": [ + { + "description": "any array is valid", + "data": [ 1, "foo", true ], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schema (false)", + "schema": {"items": false}, + "tests": [ + { + "description": "any non-empty array is invalid", + "data": [ 1, "foo", true ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items with boolean schemas", + "schema": { + "items": [true, false] + }, + "tests": [ + { + "description": "array with one item is valid", + "data": [ 1 ], + "valid": true + }, + { + "description": "array with two items is invalid", + "data": [ 1, "foo" ], + "valid": false + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "items and subitems", + "schema": { + "definitions": { + "item": { + "type": "array", + "additionalItems": false, + "items": [ + { "$ref": "#/definitions/sub-item" }, + { "$ref": "#/definitions/sub-item" } + ] + }, + "sub-item": { + "type": "object", + "required": ["foo"] + } + }, + "type": "array", + "additionalItems": false, + "items": [ + { "$ref": "#/definitions/item" }, + { "$ref": "#/definitions/item" }, + { "$ref": "#/definitions/item" } + ] + }, + "tests": [ + { + "description": "valid items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": true + }, + { + "description": "too many items", + "data": [ + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "too many sub-items", + "data": [ + [ {"foo": null}, {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong item", + "data": [ + {"foo": null}, + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "wrong sub-item", + "data": [ + [ {}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ], + [ {"foo": null}, {"foo": null} ] + ], + "valid": false + }, + { + "description": "fewer items is valid", + "data": [ + [ {"foo": null} ], + [ {"foo": null} ] + ], + "valid": true + } + ] + }, + { + "description": "nested items", + "schema": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + } + }, + "tests": [ + { + "description": "valid nested array", + "data": [[[[1]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": true + }, + { + "description": "nested array with invalid type", + "data": [[[["1"]], [[2],[3]]], [[[4], [5], [6]]]], + "valid": false + }, + { + "description": "not deep enough", + "data": [[[1], [2],[3]], [[4], [5], [6]]], + "valid": false + } + ] + }, + { + "description": "single-form items with null instance elements", + "schema": { + "items": { + "type": "null" + } + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + }, + { + "description": "array-form items with null instance elements", + "schema": { + "items": [ + { + "type": "null" + } + ] + }, + "tests": [ + { + "description": "allows null elements", + "data": [ null ], + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json new file mode 100644 index 000000000..f0c36ab28 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxItems.json @@ -0,0 +1,44 @@ +[ + { + "description": "maxItems validation", + "schema": {"maxItems": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "foobar", + "valid": true + } + ] + }, + { + "description": "maxItems validation with a decimal", + "schema": {"maxItems": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": [1], + "valid": true + }, + { + "description": "too long is invalid", + "data": [1, 2, 3], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json new file mode 100644 index 000000000..be60c5407 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxLength.json @@ -0,0 +1,49 @@ +[ + { + "description": "maxLength validation", + "schema": {"maxLength": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + }, + { + "description": "two graphemes is long enough", + "data": "\uD83D\uDCA9\uD83D\uDCA9", + "valid": true + } + ] + }, + { + "description": "maxLength validation with a decimal", + "schema": {"maxLength": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": "f", + "valid": true + }, + { + "description": "too long is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json new file mode 100644 index 000000000..acec14209 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maxProperties.json @@ -0,0 +1,70 @@ +[ + { + "description": "maxProperties validation", + "schema": {"maxProperties": 2}, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [1, 2, 3], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "maxProperties validation with a decimal", + "schema": {"maxProperties": 2.0}, + "tests": [ + { + "description": "shorter is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too long is invalid", + "data": {"foo": 1, "bar": 2, "baz": 3}, + "valid": false + } + ] + }, + { + "description": "maxProperties = 0 means the object is empty", + "schema": { "maxProperties": 0 }, + "tests": [ + { + "description": "no properties is valid", + "data": {}, + "valid": true + }, + { + "description": "one property is invalid", + "data": { "foo": 1 }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json new file mode 100644 index 000000000..c3e727069 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/maximum.json @@ -0,0 +1,54 @@ +[ + { + "description": "maximum validation", + "schema": {"maximum": 3.0}, + "tests": [ + { + "description": "below the maximum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 3.0, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 3.5, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "maximum validation with unsigned integer", + "schema": {"maximum": 300}, + "tests": [ + { + "description": "below the maximum is valid", + "data": 299.97, + "valid": true + }, + { + "description": "boundary point integer is valid", + "data": 300, + "valid": true + }, + { + "description": "boundary point float is valid", + "data": 300.00, + "valid": true + }, + { + "description": "above the maximum is invalid", + "data": 300.5, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json new file mode 100644 index 000000000..d3b18720d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minItems.json @@ -0,0 +1,44 @@ +[ + { + "description": "minItems validation", + "schema": {"minItems": 1}, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "exact length is valid", + "data": [1], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + }, + { + "description": "ignores non-arrays", + "data": "", + "valid": true + } + ] + }, + { + "description": "minItems validation with a decimal", + "schema": {"minItems": 1.0}, + "tests": [ + { + "description": "longer is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "too short is invalid", + "data": [], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json new file mode 100644 index 000000000..23c68fe3f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minLength.json @@ -0,0 +1,49 @@ +[ + { + "description": "minLength validation", + "schema": {"minLength": 2}, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "exact length is valid", + "data": "fo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 1, + "valid": true + }, + { + "description": "one grapheme is not long enough", + "data": "\uD83D\uDCA9", + "valid": false + } + ] + }, + { + "description": "minLength validation with a decimal", + "schema": {"minLength": 2.0}, + "tests": [ + { + "description": "longer is valid", + "data": "foo", + "valid": true + }, + { + "description": "too short is invalid", + "data": "f", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json new file mode 100644 index 000000000..416e02010 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minProperties.json @@ -0,0 +1,64 @@ +[ + { + "description": "minProperties validation", + "schema": {"minProperties": 1}, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "exact length is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "minProperties validation with a decimal", + "schema": {"minProperties": 1.0}, + "tests": [ + { + "description": "longer is valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "too short is invalid", + "data": {}, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json new file mode 100644 index 000000000..21ae50e0e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/minimum.json @@ -0,0 +1,69 @@ +[ + { + "description": "minimum validation", + "schema": {"minimum": 1.1}, + "tests": [ + { + "description": "above the minimum is valid", + "data": 2.6, + "valid": true + }, + { + "description": "boundary point is valid", + "data": 1.1, + "valid": true + }, + { + "description": "below the minimum is invalid", + "data": 0.6, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + }, + { + "description": "minimum validation with signed integer", + "schema": {"minimum": -2}, + "tests": [ + { + "description": "negative above the minimum is valid", + "data": -1, + "valid": true + }, + { + "description": "positive above the minimum is valid", + "data": 0, + "valid": true + }, + { + "description": "boundary point is valid", + "data": -2, + "valid": true + }, + { + "description": "boundary point with float is valid", + "data": -2.0, + "valid": true + }, + { + "description": "float below the minimum is invalid", + "data": -2.0001, + "valid": false + }, + { + "description": "int below the minimum is invalid", + "data": -3, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "x", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json new file mode 100644 index 000000000..da0ce0830 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/multipleOf.json @@ -0,0 +1,87 @@ +[ + { + "description": "by int", + "schema": {"multipleOf": 2}, + "tests": [ + { + "description": "int by int", + "data": 10, + "valid": true + }, + { + "description": "int by int fail", + "data": 7, + "valid": false + }, + { + "description": "ignores non-numbers", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "by number", + "schema": {"multipleOf": 1.5}, + "tests": [ + { + "description": "zero is multiple of anything", + "data": 0, + "valid": true + }, + { + "description": "4.5 is multiple of 1.5", + "data": 4.5, + "valid": true + }, + { + "description": "-4.5 is multiple of 1.5", + "data": -4.5, + "valid": true + }, + { + "description": "35 is not multiple of 1.5", + "data": 35, + "valid": false + } + ] + }, + { + "description": "by small number", + "schema": {"multipleOf": 0.0001}, + "tests": [ + { + "description": "0.0075 is multiple of 0.0001", + "data": 0.0075, + "valid": true + }, + { + "description": "0.00751 is not multiple of 0.0001", + "data": 0.00751, + "valid": false + } + ] + }, + { + "description": "float division = inf", + "schema": {"type": "integer", "multipleOf": 0.123456789}, + "tests": [ + { + "description": "always invalid, but naive implementations may raise an overflow error", + "data": 1e308, + "valid": false + } + ] + }, + { + "description": "small multiple of large integer", + "schema": {"type": "integer", "multipleOf": 1e-8}, + "tests": [ + { + "description": "any integer is a multiple of 1e-8", + "data": 12391239123, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json new file mode 100644 index 000000000..b46c4ed05 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/not.json @@ -0,0 +1,259 @@ +[ + { + "description": "not", + "schema": { + "not": {"type": "integer"} + }, + "tests": [ + { + "description": "allowed", + "data": "foo", + "valid": true + }, + { + "description": "disallowed", + "data": 1, + "valid": false + } + ] + }, + { + "description": "not multiple types", + "schema": { + "not": {"type": ["integer", "boolean"]} + }, + "tests": [ + { + "description": "valid", + "data": "foo", + "valid": true + }, + { + "description": "mismatch", + "data": 1, + "valid": false + }, + { + "description": "other mismatch", + "data": true, + "valid": false + } + ] + }, + { + "description": "not more complex schema", + "schema": { + "not": { + "type": "object", + "properties": { + "foo": { + "type": "string" + } + } + } + }, + "tests": [ + { + "description": "match", + "data": 1, + "valid": true + }, + { + "description": "other match", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "mismatch", + "data": {"foo": "bar"}, + "valid": false + } + ] + }, + { + "description": "forbidden property", + "schema": { + "properties": { + "foo": { + "not": {} + } + } + }, + "tests": [ + { + "description": "property present", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "property absent", + "data": {"bar": 1, "baz": 2}, + "valid": true + } + ] + }, + { + "description": "forbid everything with empty schema", + "schema": { "not": {} }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "forbid everything with boolean schema true", + "schema": { "not": true }, + "tests": [ + { + "description": "number is invalid", + "data": 1, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "boolean true is invalid", + "data": true, + "valid": false + }, + { + "description": "boolean false is invalid", + "data": false, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + }, + { + "description": "object is invalid", + "data": {"foo": "bar"}, + "valid": false + }, + { + "description": "empty object is invalid", + "data": {}, + "valid": false + }, + { + "description": "array is invalid", + "data": ["foo"], + "valid": false + }, + { + "description": "empty array is invalid", + "data": [], + "valid": false + } + ] + }, + { + "description": "allow everything with boolean schema false", + "schema": { "not": false }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "boolean true is valid", + "data": true, + "valid": true + }, + { + "description": "boolean false is valid", + "data": false, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + }, + { + "description": "array is valid", + "data": ["foo"], + "valid": true + }, + { + "description": "empty array is valid", + "data": [], + "valid": true + } + ] + }, + { + "description": "double negation", + "schema": { "not": { "not": {} } }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json new file mode 100644 index 000000000..c30a65c0d --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/oneOf.json @@ -0,0 +1,274 @@ +[ + { + "description": "oneOf", + "schema": { + "oneOf": [ + { + "type": "integer" + }, + { + "minimum": 2 + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": 1, + "valid": true + }, + { + "description": "second oneOf valid", + "data": 2.5, + "valid": true + }, + { + "description": "both oneOf valid", + "data": 3, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": 1.5, + "valid": false + } + ] + }, + { + "description": "oneOf with base schema", + "schema": { + "type": "string", + "oneOf" : [ + { + "minLength": 2 + }, + { + "maxLength": 4 + } + ] + }, + "tests": [ + { + "description": "mismatch base schema", + "data": 3, + "valid": false + }, + { + "description": "one oneOf valid", + "data": "foobar", + "valid": true + }, + { + "description": "both oneOf valid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all true", + "schema": {"oneOf": [true, true, true]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, one true", + "schema": {"oneOf": [true, false, false]}, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "oneOf with boolean schemas, more than one true", + "schema": {"oneOf": [true, true, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf with boolean schemas, all false", + "schema": {"oneOf": [false, false, false]}, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "oneOf complex types", + "schema": { + "oneOf": [ + { + "properties": { + "bar": {"type": "integer"} + }, + "required": ["bar"] + }, + { + "properties": { + "foo": {"type": "string"} + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid (complex)", + "data": {"bar": 2}, + "valid": true + }, + { + "description": "second oneOf valid (complex)", + "data": {"foo": "baz"}, + "valid": true + }, + { + "description": "both oneOf valid (complex)", + "data": {"foo": "baz", "bar": 2}, + "valid": false + }, + { + "description": "neither oneOf valid (complex)", + "data": {"foo": 2, "bar": "quux"}, + "valid": false + } + ] + }, + { + "description": "oneOf with empty schema", + "schema": { + "oneOf": [ + { "type": "number" }, + {} + ] + }, + "tests": [ + { + "description": "one valid - valid", + "data": "foo", + "valid": true + }, + { + "description": "both valid - invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "oneOf with required", + "schema": { + "type": "object", + "oneOf": [ + { "required": ["foo", "bar"] }, + { "required": ["foo", "baz"] } + ] + }, + "tests": [ + { + "description": "both invalid - invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "first valid - valid", + "data": {"foo": 1, "bar": 2}, + "valid": true + }, + { + "description": "second valid - valid", + "data": {"foo": 1, "baz": 3}, + "valid": true + }, + { + "description": "both valid - invalid", + "data": {"foo": 1, "bar": 2, "baz" : 3}, + "valid": false + } + ] + }, + { + "description": "oneOf with missing optional property", + "schema": { + "oneOf": [ + { + "properties": { + "bar": true, + "baz": true + }, + "required": ["bar"] + }, + { + "properties": { + "foo": true + }, + "required": ["foo"] + } + ] + }, + "tests": [ + { + "description": "first oneOf valid", + "data": {"bar": 8}, + "valid": true + }, + { + "description": "second oneOf valid", + "data": {"foo": "foo"}, + "valid": true + }, + { + "description": "both oneOf valid", + "data": {"foo": "foo", "bar": 8}, + "valid": false + }, + { + "description": "neither oneOf valid", + "data": {"baz": "quux"}, + "valid": false + } + ] + }, + { + "description": "nested oneOf, to check validation semantics", + "schema": { + "oneOf": [ + { + "oneOf": [ + { + "type": "null" + } + ] + } + ] + }, + "tests": [ + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "anything non-null is invalid", + "data": 123, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json new file mode 100644 index 000000000..94b4a4e62 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/bignum.json @@ -0,0 +1,93 @@ +[ + { + "description": "integer", + "schema": { "type": "integer" }, + "tests": [ + { + "description": "a bignum is an integer", + "data": 12345678910111213141516171819202122232425262728293031, + "valid": true + }, + { + "description": "a negative bignum is an integer", + "data": -12345678910111213141516171819202122232425262728293031, + "valid": true + } + ] + }, + { + "description": "number", + "schema": { "type": "number" }, + "tests": [ + { + "description": "a bignum is a number", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": true + }, + { + "description": "a negative bignum is a number", + "data": -98249283749234923498293171823948729348710298301928331, + "valid": true + } + ] + }, + { + "description": "string", + "schema": { "type": "string" }, + "tests": [ + { + "description": "a bignum is not a string", + "data": 98249283749234923498293171823948729348710298301928331, + "valid": false + } + ] + }, + { + "description": "maximum integer comparison", + "schema": { "maximum": 18446744073709551615 }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision", + "schema": { + "exclusiveMaximum": 972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for high numbers", + "data": 972783798187987123879878123.188781371, + "valid": false + } + ] + }, + { + "description": "minimum integer comparison", + "schema": { "minimum": -18446744073709551615 }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -18446744073709551600, + "valid": true + } + ] + }, + { + "description": "float comparison with high precision on negative numbers", + "schema": { + "exclusiveMinimum": -972783798187987123879878123.18878137 + }, + "tests": [ + { + "description": "comparison works for very negative numbers", + "data": -972783798187987123879878123.188781371, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json new file mode 100644 index 000000000..3f5a7430b --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/content.json @@ -0,0 +1,77 @@ +[ + { + "description": "validation of string-encoded content based on media type", + "schema": { + "contentMediaType": "application/json" + }, + "tests": [ + { + "description": "a valid JSON document", + "data": "{\"foo\": \"bar\"}", + "valid": true + }, + { + "description": "an invalid JSON document", + "data": "{:}", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary string-encoding", + "schema": { + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64 string", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "an invalid base64 string (% is not a valid character)", + "data": "eyJmb28iOi%iYmFyIn0K", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + }, + { + "description": "validation of binary-encoded media type documents", + "schema": { + "contentMediaType": "application/json", + "contentEncoding": "base64" + }, + "tests": [ + { + "description": "a valid base64-encoded JSON document", + "data": "eyJmb28iOiAiYmFyIn0K", + "valid": true + }, + { + "description": "a validly-encoded invalid JSON document", + "data": "ezp9Cg==", + "valid": false + }, + { + "description": "an invalid base64 string that is valid JSON", + "data": "{}", + "valid": false + }, + { + "description": "ignores non-strings", + "data": 100, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json new file mode 100644 index 000000000..8ff537362 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/cross-draft.json @@ -0,0 +1,25 @@ +[ + { + "description": "refs to future drafts are processed as future drafts", + "schema": { + "type": "object", + "allOf": [ + { "properties": { "foo": true } }, + { "$ref": "http://localhost:1234/draft2019-09/dependentRequired.json" } + ] + }, + "tests": [ + { + "description": "missing bar is invalid", + "comment": "if the implementation is not processing the $ref as a 2019-09 schema, this test will fail", + "data": {"foo": "any value"}, + "valid": false + }, + { + "description": "present bar is valid", + "data": {"foo": "any value", "bar": "also any value"}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json new file mode 100644 index 000000000..c4886aaa8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/ecmascript-regex.json @@ -0,0 +1,552 @@ +[ + { + "description": "ECMA 262 regex $ does not match trailing newline", + "schema": { + "type": "string", + "pattern": "^abc$" + }, + "tests": [ + { + "description": "matches in Python, but not in ECMA 262", + "data": "abc\\n", + "valid": false + }, + { + "description": "matches", + "data": "abc", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex converts \\t to horizontal tab", + "schema": { + "type": "string", + "pattern": "^\\t$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\t", + "valid": false + }, + { + "description": "matches", + "data": "\u0009", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and upper letter", + "schema": { + "type": "string", + "pattern": "^\\cC$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cC", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 regex escapes control codes with \\c and lower letter", + "schema": { + "type": "string", + "pattern": "^\\cc$" + }, + "tests": [ + { + "description": "does not match", + "data": "\\cc", + "valid": false + }, + { + "description": "matches", + "data": "\u0003", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\d matches ascii digits only", + "schema": { + "type": "string", + "pattern": "^\\d$" + }, + "tests": [ + { + "description": "ASCII zero matches", + "data": "0", + "valid": true + }, + { + "description": "NKO DIGIT ZERO does not match (unlike e.g. Python)", + "data": "߀", + "valid": false + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) does not match", + "data": "\u07c0", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\D matches everything but ascii digits", + "schema": { + "type": "string", + "pattern": "^\\D$" + }, + "tests": [ + { + "description": "ASCII zero does not match", + "data": "0", + "valid": false + }, + { + "description": "NKO DIGIT ZERO matches (unlike e.g. Python)", + "data": "߀", + "valid": true + }, + { + "description": "NKO DIGIT ZERO (as \\u escape) matches", + "data": "\u07c0", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\w matches ascii letters only", + "schema": { + "type": "string", + "pattern": "^\\w$" + }, + "tests": [ + { + "description": "ASCII 'a' matches", + "data": "a", + "valid": true + }, + { + "description": "latin-1 e-acute does not match (unlike e.g. Python)", + "data": "é", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\W matches everything but ascii letters", + "schema": { + "type": "string", + "pattern": "^\\W$" + }, + "tests": [ + { + "description": "ASCII 'a' does not match", + "data": "a", + "valid": false + }, + { + "description": "latin-1 e-acute matches (unlike e.g. Python)", + "data": "é", + "valid": true + } + ] + }, + { + "description": "ECMA 262 \\s matches whitespace", + "schema": { + "type": "string", + "pattern": "^\\s$" + }, + "tests": [ + { + "description": "ASCII space matches", + "data": " ", + "valid": true + }, + { + "description": "Character tabulation matches", + "data": "\t", + "valid": true + }, + { + "description": "Line tabulation matches", + "data": "\u000b", + "valid": true + }, + { + "description": "Form feed matches", + "data": "\u000c", + "valid": true + }, + { + "description": "latin-1 non-breaking-space matches", + "data": "\u00a0", + "valid": true + }, + { + "description": "zero-width whitespace matches", + "data": "\ufeff", + "valid": true + }, + { + "description": "line feed matches (line terminator)", + "data": "\u000a", + "valid": true + }, + { + "description": "paragraph separator matches (line terminator)", + "data": "\u2029", + "valid": true + }, + { + "description": "EM SPACE matches (Space_Separator)", + "data": "\u2003", + "valid": true + }, + { + "description": "Non-whitespace control does not match", + "data": "\u0001", + "valid": false + }, + { + "description": "Non-whitespace does not match", + "data": "\u2013", + "valid": false + } + ] + }, + { + "description": "ECMA 262 \\S matches everything but whitespace", + "schema": { + "type": "string", + "pattern": "^\\S$" + }, + "tests": [ + { + "description": "ASCII space does not match", + "data": " ", + "valid": false + }, + { + "description": "Character tabulation does not match", + "data": "\t", + "valid": false + }, + { + "description": "Line tabulation does not match", + "data": "\u000b", + "valid": false + }, + { + "description": "Form feed does not match", + "data": "\u000c", + "valid": false + }, + { + "description": "latin-1 non-breaking-space does not match", + "data": "\u00a0", + "valid": false + }, + { + "description": "zero-width whitespace does not match", + "data": "\ufeff", + "valid": false + }, + { + "description": "line feed does not match (line terminator)", + "data": "\u000a", + "valid": false + }, + { + "description": "paragraph separator does not match (line terminator)", + "data": "\u2029", + "valid": false + }, + { + "description": "EM SPACE does not match (Space_Separator)", + "data": "\u2003", + "valid": false + }, + { + "description": "Non-whitespace control matches", + "data": "\u0001", + "valid": true + }, + { + "description": "Non-whitespace matches", + "data": "\u2013", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with pattern", + "schema": { "pattern": "\\p{Letter}cole" }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "\\w in patterns matches [A-Za-z0-9_], not unicode letters", + "schema": { "pattern": "\\wcole" }, + "tests": [ + { + "description": "ascii character in json string", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": "LES HIVERS DE MON ENFANCE ÉTAIENT DES SAISONS LONGUES, LONGUES. NOUS VIVIONS EN TROIS LIEUX: L'ÉCOLE, L'ÉGLISE ET LA PATINOIRE; MAIS LA VRAIE VIE ÉTAIT SUR LA PATINOIRE.", + "valid": false + } + ] + }, + { + "description": "pattern with ASCII ranges", + "schema": { "pattern": "[a-z]cole" }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'école, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": "Les hivers de mon enfance étaient des saisons longues, longues. Nous vivions en trois lieux: l'\u00e9cole, l'église et la patinoire; mais la vraie vie était sur la patinoire.", + "valid": false + }, + { + "description": "ascii characters match", + "data": "Les hivers de mon enfance etaient des saisons longues, longues. Nous vivions en trois lieux: l'ecole, l'eglise et la patinoire; mais la vraie vie etait sur la patinoire.", + "valid": true + } + ] + }, + { + "description": "\\d in pattern matches [0-9], not unicode digits", + "schema": { "pattern": "^\\d+$" }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": false + } + ] + }, + { + "description": "pattern with non-ASCII digits", + "schema": { "pattern": "^\\p{digit}+$" }, + "tests": [ + { + "description": "ascii digits", + "data": "42", + "valid": true + }, + { + "description": "ascii non-digits", + "data": "-%#", + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": "৪২", + "valid": true + } + ] + }, + { + "description": "patterns always use unicode semantics with patternProperties", + "schema": { + "type": "object", + "patternProperties": { + "\\p{Letter}cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "\\w in patternProperties matches [A-Za-z0-9_], not unicode letters", + "schema": { + "type": "object", + "patternProperties": { + "\\wcole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii character in json string", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + }, + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode matching is case-sensitive", + "data": { "L'ÉCOLE": "PAS DE VRAIE VIE" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with ASCII ranges", + "schema": { + "type": "object", + "patternProperties": { + "[a-z]cole": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "literal unicode character in json string", + "data": { "l'école": "pas de vraie vie" }, + "valid": false + }, + { + "description": "unicode character in hex format in string", + "data": { "l'\u00e9cole": "pas de vraie vie" }, + "valid": false + }, + { + "description": "ascii characters match", + "data": { "l'ecole": "pas de vraie vie" }, + "valid": true + } + ] + }, + { + "description": "\\d in patternProperties matches [0-9], not unicode digits", + "schema": { + "type": "object", + "patternProperties": { + "^\\d+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": false + } + ] + }, + { + "description": "patternProperties with non-ASCII digits", + "schema": { + "type": "object", + "patternProperties": { + "^\\p{digit}+$": true + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "ascii digits", + "data": { "42": "life, the universe, and everything" }, + "valid": true + }, + { + "description": "ascii non-digits", + "data": { "-%#": "spending the year dead for tax reasons" }, + "valid": false + }, + { + "description": "non-ascii digits (BENGALI DIGIT FOUR, BENGALI DIGIT TWO)", + "data": { "৪২": "khajit has wares if you have coin" }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json new file mode 100644 index 000000000..52ff9827c --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/float-overflow.json @@ -0,0 +1,13 @@ +[ + { + "description": "all integers are multiples of 0.5, if overflow is handled", + "schema": {"type": "integer", "multipleOf": 0.5}, + "tests": [ + { + "description": "valid if optional overflow handling is implemented", + "data": 1e308, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json new file mode 100644 index 000000000..07a4e9195 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date-time.json @@ -0,0 +1,247 @@ +[ + { + "description": "validation of date-time strings", + "schema": { "format": "date-time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date-time string", + "data": "1963-06-19T08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid date-time string without second fraction", + "data": "1963-06-19T08:30:06Z", + "valid": true + }, + { + "description": "a valid date-time string with plus offset", + "data": "1937-01-01T12:00:27.87+00:20", + "valid": true + }, + { + "description": "a valid date-time string with minus offset", + "data": "1990-12-31T15:59:50.123-08:00", + "valid": true + }, + { + "description": "a valid date-time with a leap second, UTC", + "data": "1998-12-31T23:59:60Z", + "valid": true + }, + { + "description": "a valid date-time with a leap second, with minus offset", + "data": "1998-12-31T15:59:60.123-08:00", + "valid": true + }, + { + "description": "an invalid date-time past leap second, UTC", + "data": "1998-12-31T23:59:61Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong minute, UTC", + "data": "1998-12-31T23:58:60Z", + "valid": false + }, + { + "description": "an invalid date-time with leap second on a wrong hour, UTC", + "data": "1998-12-31T22:59:60Z", + "valid": false + }, + { + "description": "an invalid hour in date-time string", + "data": "1990-12-31T24:00:00Z", + "valid": false + }, + { + "description": "an invalid minute in date-time string", + "data": "1990-12-31T15:60:00Z", + "valid": false + }, + { + "description": "an invalid offset minute in date-time string", + "data": "1990-12-31T10:00:00+10:60", + "valid": false + }, + { + "description": "an invalid day in date-time string", + "data": "1990-02-31T15:59:59.123-08:00", + "valid": false + }, + { + "description": "an invalid offset in date-time string", + "data": "1990-12-31T15:59:59-24:00", + "valid": false + }, + { + "description": "an invalid closing Z after time-zone offset", + "data": "1963-06-19T08:30:06.28123+01:00Z", + "valid": false + }, + { + "description": "an invalid date-time string", + "data": "06/19/1963 08:30:06 PST", + "valid": false + }, + { + "description": "case-insensitive T and Z", + "data": "1963-06-19t08:30:06.283185z", + "valid": true + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350T01:01:01", + "valid": false + }, + { + "description": "invalid non-padded month dates", + "data": "1963-6-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-padded day dates", + "data": "1963-06-1T08:30:06.283185Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in date portion", + "data": "1963-06-1৪T00:00:00Z", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in time portion", + "data": "1963-06-11T0৪:00:00Z", + "valid": false + }, + { + "description": "invalid extended year", + "data": "+11963-06-19T08:30:06.283185Z", + "valid": false + }, + { + "description": "a numeric offset without minutes is invalid", + "data": "1985-04-12T23:20:50+01", + "valid": false + }, + { + "description": "hour 24 is invalid even with a leap second", + "data": "2016-12-31T24:59:60+01:00", + "valid": false + }, + { + "description": "a second fraction of fifteen nines is valid", + "data": "1985-04-12T00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a trailing newline is invalid", + "data": "1985-04-12T23:20:50Z\n", + "valid": false + }, + { + "description": "an invalid date-time string without seconds", + "data": "1985-04-12T23:20Z", + "valid": false + }, + { + "description": "an invalid date-time string with trailing content after the offset", + "data": "1985-04-12T23:20:50Ztail", + "valid": false + }, + { + "description": "an invalid minute in date-time string, with a numeric offset", + "data": "1985-04-12T23:60:00+00:01", + "valid": false + }, + { + "description": "a valid date-time string with 28 days in February (normal)", + "data": "2021-02-28T00:00:00Z", + "valid": true + }, + { + "description": "an invalid date-time string with 30 days in February (leap)", + "data": "2020-02-30T00:00:00Z", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29T00:00:00Z", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 0100 is not a leap year", + "data": "0100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "data": "0400-02-29T00:00:00Z", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "data": "2100-02-29T00:00:00Z", + "valid": false + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "1985-04-12T23:20:50.Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "1985-04-12T23:20:50,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": " 1985-04-12T23:20:50Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid date-time", + "comment": "RFC 3339 section 5.6: no production of date-time emits whitespace", + "data": "1985-04-12T23:20:50Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json new file mode 100644 index 000000000..4a3847b4f --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/date.json @@ -0,0 +1,420 @@ +[ + { + "description": "validation of date strings", + "schema": { "format": "date" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid date string", + "data": "1963-06-19", + "valid": true + }, + { + "description": "a valid date string with 31 days in January", + "data": "2020-01-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in January", + "data": "2020-01-32", + "valid": false + }, + { + "description": "a valid date string with 28 days in February (normal)", + "data": "2021-02-28", + "valid": true + }, + { + "description": "a invalid date string with 30 days in February (leap)", + "data": "2020-02-30", + "valid": false + }, + { + "description": "a valid date string with 31 days in March", + "data": "2020-03-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in March", + "data": "2020-03-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in April", + "data": "2020-04-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in April", + "data": "2020-04-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in May", + "data": "2020-05-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in May", + "data": "2020-05-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in June", + "data": "2020-06-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in June", + "data": "2020-06-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in July", + "data": "2020-07-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in July", + "data": "2020-07-32", + "valid": false + }, + { + "description": "a valid date string with 31 days in August", + "data": "2020-08-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in August", + "data": "2020-08-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in September", + "data": "2020-09-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in September", + "data": "2020-09-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in October", + "data": "2020-10-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in October", + "data": "2020-10-32", + "valid": false + }, + { + "description": "a valid date string with 30 days in November", + "data": "2020-11-30", + "valid": true + }, + { + "description": "a invalid date string with 31 days in November", + "data": "2020-11-31", + "valid": false + }, + { + "description": "a valid date string with 31 days in December", + "data": "2020-12-31", + "valid": true + }, + { + "description": "a invalid date string with 32 days in December", + "data": "2020-12-32", + "valid": false + }, + { + "description": "an invalid date string", + "data": "06/19/1963", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "2013-350", + "valid": false + }, + { + "description": "non-padded month dates are not valid", + "data": "1998-1-20", + "valid": false + }, + { + "description": "non-padded day dates are not valid", + "data": "1998-01-1", + "valid": false + }, + { + "description": "invalid month", + "data": "1998-13-01", + "valid": false + }, + { + "description": "2021 is not a leap year", + "data": "2021-02-29", + "valid": false + }, + { + "description": "2020 is a leap year", + "data": "2020-02-29", + "valid": true + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1963-06-1৪", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in month field", + "data": "2020-0৪-01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: YYYYMMDD without dashes (2023-03-28)", + "data": "20230328", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number implicit day of week (2023-01-02)", + "data": "2023-W01", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number with day of week (2023-03-28)", + "data": "2023-W13-2", + "valid": false + }, + { + "description": "ISO8601 / non-RFC3339: week number rollover to next year (2023-01-01)", + "data": "2022W527", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "century year 0100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 100 % 100 == 0 but 100 % 400 != 0", + "data": "0100-02-29", + "valid": false + }, + { + "description": "century year 0400 is a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — 400-year cycle boundary", + "data": "0400-02-29", + "valid": true + }, + { + "description": "century year 2100 is not a leap year", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#appendix-C — future century year", + "data": "2100-02-29", + "valid": false + }, + { + "description": "invalid: leading whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": " 2024-01-15", + "valid": false + }, + { + "description": "invalid: trailing whitespace is not permitted", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — full-date grammar does not include whitespace", + "data": "2024-01-15 ", + "valid": false + }, + { + "description": "invalid: month 00 is not valid per date-month range 01-12", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-month = 2DIGIT ; 01-12", + "data": "2024-00-15", + "valid": false + }, + { + "description": "invalid: day 00 is not valid per date-mday minimum of 01", + "comment": "https://www.rfc-editor.org/rfc/rfc3339#section-5.6 — date-mday = 2DIGIT ; 01-28/29/30/31", + "data": "2024-01-00", + "valid": false + }, + { + "description": "invalid: empty string", + "data": "", + "valid": false + }, + { + "description": "invalid: embedded whitespace between year and month", + "data": "2020 -01-01", + "valid": false + }, + { + "description": "invalid: trailing character after valid full-date", + "data": "2020-01-01X", + "valid": false + }, + { + "description": "invalid: trailing Z after full-date", + "data": "2020-01-01Z", + "valid": false + }, + { + "description": "invalid: full-date followed by space and time component", + "data": "2020-01-01 00:00:00Z", + "valid": false + }, + { + "description": "valid: four-digit year 0001", + "data": "0001-01-01", + "valid": true + }, + { + "description": "invalid: two-digit year (N-2 digits)", + "data": "20-01-01", + "valid": false + }, + { + "description": "invalid: three-digit year (N-1 digits)", + "data": "998-01-01", + "valid": false + }, + { + "description": "invalid: five-digit year (N+1 digits)", + "data": "12020-01-01", + "valid": false + }, + { + "description": "invalid: positive sign prefix on year", + "data": "+2020-01-01", + "valid": false + }, + { + "description": "invalid: negative sign prefix on year", + "data": "-2020-01-01", + "valid": false + }, + { + "description": "invalid: non-ASCII Bengali digit in year field", + "data": "২020-01-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in year field", + "data": "YYYY-01-01", + "valid": false + }, + { + "description": "invalid: three-digit month (N+1 digits)", + "data": "2020-001-01", + "valid": false + }, + { + "description": "invalid: alphabetic characters in month field", + "data": "2020-MM-01", + "valid": false + }, + { + "description": "invalid: three-digit day (N+1 digits)", + "data": "2020-01-001", + "valid": false + }, + { + "description": "invalid: alphabetic characters in day field", + "data": "2020-01-DD", + "valid": false + }, + { + "description": "invalid: colon separators", + "data": "2020:01:01", + "valid": false + }, + { + "description": "invalid: dot separators", + "data": "2020.01.01", + "valid": false + }, + { + "description": "invalid: space separators", + "data": "2020 01 01", + "valid": false + }, + { + "description": "invalid: mixed slash and hyphen separators", + "data": "2020-01/01", + "valid": false + }, + { + "description": "invalid: duplicated first hyphen", + "data": "2020--01-01", + "valid": false + }, + { + "description": "invalid: duplicated second hyphen", + "data": "2020-01--01", + "valid": false + }, + { + "description": "valid: date inside the Julian to Gregorian reform gap", + "data": "1582-10-10", + "valid": true + }, + { + "description": "invalid: year field numerically larger than a 32-bit signed integer", + "data": "2147483648-01-01", + "valid": false + }, + { + "description": "invalid: character just above '9' as the second digit of the day", + "data": "2020-01-0:", + "valid": false + }, + { + "description": "invalid: non-ASCII digit leaving ten UTF-8 bytes but eight characters", + "data": "\u09e80-01-01", + "valid": false + }, + { + "description": "invalid: en dash separators instead of hyphen-minus", + "data": "2020\u201301\u201301", + "valid": false + }, + { + "description": "invalid: NUL character after an otherwise valid full-date", + "data": "2020-01-01\u0000", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json new file mode 100644 index 000000000..bf3fae7f2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ecmascript-regex.json @@ -0,0 +1,98 @@ +[ + { + "description": "\\a is not an ECMA 262 control escape", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "when used as a pattern", + "data": "\\a", + "valid": false + } + ] + }, + { + "description": "Python-specific regular expression syntax is not valid ECMA 262", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "Python named group (?P...) is not ECMA 262", + "data": "(?Px)", + "valid": false + }, + { + "description": "Python named backreference (?P=name) is not ECMA 262", + "data": "(?Pa)(?P=n)", + "valid": false + }, + { + "description": "inline comment group (?#...) is not ECMA 262", + "data": "(?#comment)a", + "valid": false + } + ] + }, + { + "description": "global inline flag groups are not valid ECMA 262", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "a single global inline flag (?i)", + "data": "(?i)abc", + "valid": false + }, + { + "description": "multiple global inline flags (?ims)", + "data": "(?ims)abc", + "valid": false + } + ] + }, + { + "description": "ECMA 262 named groups and backreferences are valid", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "an ECMA 262 named group (?...)", + "data": "(?x)", + "valid": true + }, + { + "description": "an ECMA 262 named backreference \\k", + "data": "(?a)\\k", + "valid": true + } + ] + }, + { + "description": "ECMA 262 lookbehind is valid, including variable width", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "a variable-width lookbehind (ES2018)", + "data": "(?<=a+)b", + "valid": true + } + ] + }, + { + "description": "ECMA 262 character classes and escapes", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "an empty character class is valid ECMA 262", + "data": "[]", + "valid": true + }, + { + "description": "a negated empty character class is valid ECMA 262", + "data": "[^]", + "valid": true + }, + { + "description": "a control escape \\cA is valid ECMA 262", + "data": "\\cA", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json new file mode 100644 index 000000000..6ef76f3c6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/email.json @@ -0,0 +1,365 @@ +[ + { + "description": "validation of e-mail addresses", + "schema": { "format": "email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "an invalid e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "tilde in local part is valid", + "data": "te~st@example.com", + "valid": true + }, + { + "description": "tilde before local part is valid", + "data": "~test@example.com", + "valid": true + }, + { + "description": "tilde after local part is valid", + "data": "test~@example.com", + "valid": true + }, + { + "description": "a quoted string with a space in the local part is valid", + "data": "\"joe bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a double dot in the local part is valid", + "data": "\"joe..bloggs\"@example.com", + "valid": true + }, + { + "description": "a quoted string with a @ in the local part is valid", + "data": "\"joe@bloggs\"@example.com", + "valid": true + }, + { + "description": "an IPv4-address-literal after the @ is valid", + "data": "joe.bloggs@[127.0.0.1]", + "valid": true + }, + { + "description": "an IPv6-address-literal after the @ is valid", + "data": "joe.bloggs@[IPv6:::1]", + "valid": true + }, + { + "description": "dot before local part is not valid", + "data": ".test@example.com", + "valid": false + }, + { + "description": "dot after local part is not valid", + "data": "test.@example.com", + "valid": false + }, + { + "description": "two separated dots inside local part are valid", + "data": "te.s.t@example.com", + "valid": true + }, + { + "description": "two subsequent dots inside local part are not valid", + "data": "te..st@example.com", + "valid": false + }, + { + "description": "an invalid domain", + "data": "joe.bloggs@invalid=domain.com", + "valid": false + }, + { + "description": "an invalid IPv4-address-literal", + "data": "joe.bloggs@[127.0.0.300]", + "valid": false + }, + { + "description": "two email addresses is not valid", + "data": "user1@oceania.org, user2@oceania.org", + "valid": false + }, + { + "description": "full \"From\" header is invalid", + "data": "\"Winston Smith\" (Records Department)", + "valid": false + }, + { + "description": "local part is required", + "data": "@example.com", + "valid": false + }, + { + "description": "domain is required", + "data": "joe.bloggs@", + "valid": false + }, + { + "description": "unquoted space in local part is invalid", + "data": "joe bloggs@example.com", + "valid": false + }, + { + "description": "a domain that looks like an IPv4 address without brackets is valid", + "data": "test@255.255.255.255", + "valid": true + }, + { + "description": "an empty quoted string in the local part is valid", + "data": "\"\"@iana.org", + "valid": true + }, + { + "description": "a quoted string containing only a space in the local part is valid", + "data": "\" \"@iana.org", + "valid": true + }, + { + "description": "consecutive hyphens inside a domain label are valid", + "data": "test@c--n.com", + "valid": true + }, + { + "description": "a domain label starting with a digit is valid", + "data": "test@123.com", + "valid": true + }, + { + "description": "a non-ASCII character in the local part is not valid", + "data": "a\u00e9@iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in the domain is not valid", + "data": "a@\u00e9.org", + "valid": false + }, + { + "description": "a fullwidth commercial at is not a valid separator", + "data": "a\uff20iana.org", + "valid": false + }, + { + "description": "a non-ASCII character in a quoted pair is not valid", + "data": "\"test\\\u00a9\"@iana.org", + "valid": false + }, + { + "description": "a trailing line feed is not valid", + "data": "test@iana.org\n", + "valid": false + }, + { + "description": "a trailing carriage return is not valid", + "data": "test@iana.org\r", + "valid": false + }, + { + "description": "a trailing space is not valid", + "data": "a@iana.org ", + "valid": false + }, + { + "description": "a leading space is not valid", + "data": " a@iana.org", + "valid": false + }, + { + "description": "a tab in the local part is not valid", + "data": "a\tb@iana.org", + "valid": false + }, + { + "description": "a delete character in the local part is not valid", + "data": "\u007f@iana.org", + "valid": false + }, + { + "description": "a quoted string with no special characters in the local part is valid", + "data": "\"test\"@iana.org", + "valid": true + }, + { + "description": "a quoted pair in the local part is valid", + "data": "\"\\a\"@iana.org", + "valid": true + }, + { + "description": "an escaped double quote in the local part is valid", + "data": "\"\\\"\"@iana.org", + "valid": true + }, + { + "description": "an escaped backslash in the local part is valid", + "data": "\"\\\\\"@iana.org", + "valid": true + }, + { + "description": "an unescaped double quote in an unquoted local part is not valid", + "data": "test\"@iana.org", + "valid": false + }, + { + "description": "text after the closing double quote in the local part is not valid", + "data": "\"test\"test@iana.org", + "valid": false + }, + { + "description": "an unclosed double quote in the local part is not valid", + "data": "\"test@iana.org", + "valid": false + }, + { + "description": "two unquoted @ signs are not valid", + "data": "a@b@c.org", + "valid": false + }, + { + "description": "a backslash-escaped @ in an unquoted local part is not valid", + "data": "test\\@test@iana.org", + "valid": false + }, + { + "description": "an address literal in the local part position is not valid", + "data": "[1.2.3.4]@iana.org", + "valid": false + }, + { + "description": "an empty string is not valid", + "data": "", + "valid": false + }, + { + "description": "a domain label starting with a hyphen is not valid", + "data": "test@-iana.org", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "data": "test@iana-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "data": "test@.iana.org", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "test@iana.org.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "test@iana..com", + "valid": false + }, + { + "description": "a single-label domain is valid", + "data": "test@io", + "valid": true + }, + { + "description": "an unclosed address literal is not valid", + "data": "test@[1.2.3.4", + "valid": false + }, + { + "description": "empty brackets after the @ are not valid", + "data": "a@[]", + "valid": false + }, + { + "description": "an IPv4-address-literal with three octets is not valid", + "data": "a@[1.2.3]", + "valid": false + }, + { + "description": "an IPv4-address-literal with five octets is not valid", + "data": "a@[1.2.3.4.5]", + "valid": false + }, + { + "description": "bracketed content that is not an address is not valid", + "data": "test@[RFC-5322-domain-literal]", + "valid": false + }, + { + "description": "a domain label before the opening bracket is not valid", + "data": "test@a[255.255.255.255]", + "valid": false + }, + { + "description": "a lowercase IPv6 tag in an address literal is valid", + "data": "a@[ipv6:::1]", + "valid": true + }, + { + "description": "a comma in an unquoted local part is not valid", + "data": "a,b@iana.org", + "valid": false + }, + { + "description": "a semicolon in an unquoted local part is not valid", + "data": "a;b@iana.org", + "valid": false + }, + { + "description": "a colon in an unquoted local part is not valid", + "data": "a:b@iana.org", + "valid": false + }, + { + "description": "a parenthesis in an unquoted local part is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "test(comment)@iana.org", + "valid": false + }, + { + "description": "a parenthesis in the domain is not valid", + "comment": "a comment is RFC 5322 syntax, which the RFC 5321 Mailbox grammar does not have", + "data": "a@b(c).org", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json new file mode 100644 index 000000000..bd5dcdea2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/hostname.json @@ -0,0 +1,380 @@ +[ + { + "description": "validation of host names", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name", + "data": "www.example.com", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "IDN label separator", + "data": "example\uff0ecom", + "valid": false + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "starts with hyphen", + "data": "-hostname", + "valid": false + }, + { + "description": "ends with hyphen", + "data": "hostname-", + "valid": false + }, + { + "description": "contains underscore", + "data": "host_name", + "valid": false + }, + { + "description": "exceeds maximum overall length (256)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": false + }, + { + "description": "maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijk.com", + "valid": true + }, + { + "description": "exceeds maximum label length (63)", + "data": "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijkl.com", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "invalid non-ASCII KELVIN SIGN (U+212A)", + "data": "\u212Aelvin.example.com", + "valid": false + }, + { + "description": "consecutive hyphens inside a label", + "data": "a--b.com", + "valid": true + } + ] + }, + { + "description": "validation of A-label (punycode) host names", + "schema": { "format": "hostname" }, + "tests": [ + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "xn--9n2bp8q.xn--9t4b11yi5a", + "valid": true + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "xn--07jt112bpxg.xn--9t4b11yi5a", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-txk", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-zed", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "xn--hello-6bf", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--zca29lwxobi7a", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--qmbc", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "xn--chb89f", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "xn--07jceefgh4c", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--al-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-fda", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--la-0ea", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--l-gda", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "xn--ll-0ea", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--S-jib3p", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3j", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "xn--wva3je", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--A-2hc5h", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--5db1e", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "xn--4dbc5h", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--A-2hc8h", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--5db3e", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "xn--4dbc8h", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--defabc-k64e", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--k8j5u", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--bck0j", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "xn--vek778f", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngb6iyr", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "xn--ngba1o", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "xn--0-gyc", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2er09f", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--02b508i", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "xn--11b2ezcw70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "xn--11b2ezcs70k", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "xn--ngba5hb2804a", + "valid": true + }, + { + "description": "contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json new file mode 100644 index 000000000..c88bd9fc5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-email.json @@ -0,0 +1,137 @@ +[ + { + "description": "validation of an internationalized e-mail addresses", + "schema": { "format": "idn-email" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid idn e-mail (example@example.test in Hangul)", + "data": "실례@실례.테스트", + "valid": true + }, + { + "description": "an invalid e-mail/idn e-mail address", + "data": "2962", + "valid": false + }, + { + "description": "a valid e-mail address", + "data": "joe.bloggs@example.com", + "valid": true + }, + { + "description": "a non-ASCII local part with an ASCII domain is valid", + "data": "δοκιμή@example.com", + "valid": true + }, + { + "description": "a non-ASCII quoted local part is valid", + "data": "\"δοκιμή\"@example.com", + "valid": true + }, + { + "description": "a domain label that is not in Unicode NFC is valid", + "data": "user@cafe\u0301.com", + "valid": true + }, + { + "description": "a local part that is not in Unicode NFC is valid", + "data": "cafe\u0301@example.com", + "valid": true + }, + { + "description": "a C1 control in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-2 admits U+0085", + "data": "\u0085@example.com", + "valid": true + }, + { + "description": "a noncharacter in the local part is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii; https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-3 admits U+FFFF", + "data": "\uffff@example.com", + "valid": true + }, + { + "description": "a local part at the 64-octet limit is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.5.3.1 every implementation must be able to receive a local part of 64 octets", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@example.com", + "valid": true + }, + { + "description": "a fullwidth commercial at is not a local-part separator", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal ) splits on U+0040, and any IDN mapping applies only to the domain that split produces, so U+FF20 cannot become the delimiter", + "data": "user\uff20example.com", + "valid": false + }, + { + "description": "a local part with a supplementary-plane character is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc6531#section-3.3 atext =/ UTF8-non-ascii, and https://www.rfc-editor.org/rfc/rfc3629#section-4 UTF8-4 covers U+10000 and above", + "data": "\ud835\udd4f@example.com", + "valid": true + }, + { + "description": "a domain label starting with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 sub-domain = Let-dig [Ldh-str], so a label starts with a letter or digit; RFC 6531 section 3.3 only adds sub-domain =/ U-label", + "data": "δοκιμή@-example.com", + "valid": false + }, + { + "description": "a domain label ending with a hyphen is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Ldh-str = *( ALPHA / DIGIT / \"-\" ) Let-dig, so a label ends with a letter or digit", + "data": "δοκιμή@example-.com", + "valid": false + }, + { + "description": "a domain starting with a dot is not valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Domain = sub-domain *(\".\" sub-domain) admits no empty sub-domain", + "data": "δοκιμή@.example.com", + "valid": false + }, + { + "description": "a domain ending with a dot is not valid", + "data": "δοκιμή@example.com.", + "valid": false + }, + { + "description": "two subsequent dots inside the domain are not valid", + "data": "δοκιμή@example..com", + "valid": false + }, + { + "description": "domain is required", + "comment": "https://www.rfc-editor.org/rfc/rfc5321#section-4.1.2 Mailbox = Local-part \"@\" ( Domain / address-literal )", + "data": "δοκιμή@", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json new file mode 100644 index 000000000..13d223d24 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/idn-hostname.json @@ -0,0 +1,582 @@ +[ + { + "description": "validation of internationalized host names", + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid host name (example.test in Hangul)", + "data": "실례.테스트", + "valid": true + }, + { + "description": "illegal first char U+302E Hangul single dot tone mark", + "data": "〮실례.테스트", + "valid": false + }, + { + "description": "contains illegal char U+302E Hangul single dot tone mark", + "data": "실〮례.테스트", + "valid": false + }, + { + "description": "a single label of 63 characters is valid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": true + }, + { + "description": "a single label of 64 characters is too long", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "invalid label, correct Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc3492#section-7.1", + "data": "-> $1.00 <--", + "valid": false + }, + { + "description": "valid Chinese Punycode", + "comment": "https://tools.ietf.org/html/rfc5890#section-2.3.2.1 https://tools.ietf.org/html/rfc5891#section-4.4", + "data": "xn--ihqwcrb4cv8a8dqg056pqjye", + "valid": true + }, + { + "description": "invalid Punycode", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.4 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "xn--X", + "valid": false + }, + { + "description": "U-label contains \"--\" in the 3rd and 4th position", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1 https://tools.ietf.org/html/rfc5890#section-2.3.2.1", + "data": "XN--aa---o47jg78q", + "valid": false + }, + { + "description": "U-label starts with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello", + "valid": false + }, + { + "description": "U-label ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "hello-", + "valid": false + }, + { + "description": "U-label starts and ends with a dash", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.1", + "data": "-hello-", + "valid": false + }, + { + "description": "Begins with a Spacing Combining Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0903hello", + "valid": false + }, + { + "description": "Begins with a Nonspacing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0300hello", + "valid": false + }, + { + "description": "Begins with an Enclosing Mark", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.2", + "data": "\u0488hello", + "valid": false + }, + { + "description": "Exceptions that are PVALID, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u00df\u03c2\u0f0b\u3007", + "valid": true + }, + { + "description": "Exceptions that are PVALID, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u06fd\u06fe", + "valid": true + }, + { + "description": "Exceptions that are DISALLOWED, right-to-left chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6", + "data": "\u0640\u07fa", + "valid": false + }, + { + "description": "Exceptions that are DISALLOWED, left-to-right chars", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.2 https://tools.ietf.org/html/rfc5892#section-2.6 Note: The two combining marks (U+302E and U+302F) are in the middle and not at the start", + "data": "\u3031\u3032\u3033\u3034\u3035\u302e\u302f\u303b", + "valid": false + }, + { + "description": "MIDDLE DOT with no preceding 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "a\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing preceding", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "\u00b7l", + "valid": false + }, + { + "description": "MIDDLE DOT with no following 'l'", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7a", + "valid": false + }, + { + "description": "MIDDLE DOT with nothing following", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7", + "valid": false + }, + { + "description": "MIDDLE DOT with surrounding 'l's", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.3", + "data": "l\u00b7l", + "valid": true + }, + { + "description": "Greek KERAIA not followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375S", + "valid": false + }, + { + "description": "Greek KERAIA not followed by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375", + "valid": false + }, + { + "description": "Greek KERAIA followed by Greek", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.4", + "data": "\u03b1\u0375\u03b2", + "valid": true + }, + { + "description": "Hebrew GERESH not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "A\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05f3\u05d1", + "valid": false + }, + { + "description": "Hebrew GERESH preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.5", + "data": "\u05d0\u05f3\u05d1", + "valid": true + }, + { + "description": "Hebrew GERSHAYIM not preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "A\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05f4\u05d1", + "valid": false + }, + { + "description": "Hebrew GERSHAYIM preceded by Hebrew", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.6", + "data": "\u05d0\u05f4\u05d1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with no Hiragana, Katakana, or Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "def\u30fbabc", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with no other characters", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb", + "valid": false + }, + { + "description": "KATAKANA MIDDLE DOT with Hiragana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u3041", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Katakana", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u30a1", + "valid": true + }, + { + "description": "KATAKANA MIDDLE DOT with Han", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.7", + "data": "\u30fb\u4e08", + "valid": true + }, + { + "description": "Arabic-Indic digits mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u06f0", + "valid": false + }, + { + "description": "Arabic-Indic digits not mixed with Extended Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.8", + "data": "\u0628\u0660\u0628", + "valid": true + }, + { + "description": "Extended Arabic-Indic digits not mixed with Arabic-Indic digits", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.9", + "data": "\u06f00", + "valid": true + }, + { + "description": "ZERO WIDTH JOINER not preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER not preceded by anything", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u200d\u0937", + "valid": false + }, + { + "description": "ZERO WIDTH JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.2 https://www.unicode.org/review/pr-37.pdf", + "data": "\u0915\u094d\u200d\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER preceded by Virama", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937", + "valid": true + }, + { + "description": "ZERO WIDTH NON-JOINER not preceded by Virama but matches regexp", + "comment": "https://tools.ietf.org/html/rfc5891#section-4.2.3.3 https://tools.ietf.org/html/rfc5892#appendix-A.1 https://www.w3.org/TR/alreq/#h_disjoining_enforcement", + "data": "\u0628\u064a\u200c\u0628\u064a", + "valid": true + }, + { + "description": "single label", + "data": "hostname", + "valid": true + }, + { + "description": "single label with hyphen", + "data": "host-name", + "valid": true + }, + { + "description": "single label with digits", + "data": "h0stn4me", + "valid": true + }, + { + "description": "single label starting with digit", + "data": "1host", + "valid": true + }, + { + "description": "single label ending with digit", + "data": "hostnam3", + "valid": true + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "example.com\n", + "valid": false + }, + { + "description": "leading space is invalid", + "data": " example.com", + "valid": false + }, + { + "description": "space inside a label is invalid", + "data": "ex ample.com", + "valid": false + }, + { + "description": "fullwidth digits are mapped to ASCII digits", + "comment": "IdnaMappingTable.txt maps U+FF11 to 0031, so this label maps to 123", + "data": "\uff11\uff12\uff13", + "valid": true + }, + { + "description": "a zero width space is ignored by the mapping", + "comment": "IdnaMappingTable.txt lists U+200B as ignored, so this label maps to ab", + "data": "a\u200bb", + "valid": true + }, + { + "description": "an A-label with an uppercase Punycode body is valid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.3 lowercases the A-label before the round-trip test, and RFC 3492 section 5 gives A-Z and a-z the same digit-values", + "data": "xn--NXASMQ6B", + "valid": true + }, + { + "description": "the two Arabic-Indic digit blocks may appear in different labels", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.8 scopes the rule to \"this label\", so one block per label is valid", + "data": "\u0628\u0660.\u0628\u06f0", + "valid": true + }, + { + "description": "a label of only Arabic-Indic digits is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 condition 1 allows only L, R or AL as the first character, and an Arabic-Indic digit is AN", + "data": "\u0660\u0661", + "valid": false + }, + { + "description": "a non-NFC U-label is valid after UTS 46 normalisation", + "comment": "UTS 46 section 4 normalises to NFC before validating, so the decomposed form is accepted", + "data": "cafe\u0301.com", + "valid": true + }, + { + "description": "a label of 63 fullwidth characters is valid after mapping", + "comment": "UTS 46 maps each U+FF41 to an ASCII letter, so this label is 63 octets in its mapped form", + "data": "\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41\uff41", + "valid": true + }, + { + "description": "an ACE prefix produced by the mapping step is decoded as an A-label", + "comment": "UTS 46 maps U+FF58 U+FF4E to \"xn\", so the mapped label is the A-label xn--nxasmq6b", + "data": "\uff58\uff4e--nxasmq6b", + "valid": true + }, + { + "description": "zero width non-joiner must pass at every occurrence", + "comment": "https://www.rfc-editor.org/rfc/rfc5892#appendix-A.1", + "data": "\u0915\u094d\u200c\u0937x\u200cy", + "valid": false + }, + { + "description": "Bidi domain name with a digit-first label is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2 a label in a Bidi domain name must start with an L, R or AL character", + "data": "0a.\u05d0", + "valid": false + }, + { + "description": "label starting with a digit before a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "0\u0627", + "valid": false + }, + { + "description": "left-to-right label containing a right-to-left letter is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "a\u05d0", + "valid": false + }, + { + "description": "right-to-left label mixing both digit types is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5893#section-2", + "data": "\u05d00\u0660", + "valid": false + }, + { + "description": "A-label that decodes to a disallowed code point is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5892#section-2.6", + "data": "xn--7a", + "valid": false + }, + { + "description": "A-label that decodes to a Bidi rule violation is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 https://www.rfc-editor.org/rfc/rfc5893", + "data": "xn--0ca24w", + "valid": false + }, + { + "description": "a U-label whose A-label form is longer than 63 octets is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-4.2.4 the 63-octet limit is on the A-label form, not the code point count", + "data": "\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc\u00fc", + "valid": false + }, + { + "description": "empty label between two dots is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc1034#section-3.1", + "data": "a..b", + "valid": false + }, + { + "description": "a name longer than 253 characters is invalid", + "data": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "valid": false + }, + { + "description": "A-label that decodes to only ASCII is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5890#section-2.3.2.1 a U-label must have at least one non-ASCII character", + "data": "xn--example-", + "valid": false + }, + { + "description": "non-canonical Punycode that does not re-encode to itself is invalid", + "comment": "https://www.rfc-editor.org/rfc/rfc5891#section-5.4 a label decoded from Punycode must be identical to the original A-label", + "data": "xn---9uc", + "valid": false + } + ] + }, + { + "description": "validation of separators in internationalized host names", + "specification": [ + {"rfc3490": "3.1", "quote": "Whenever dots are used as label separators, the following characters MUST be recognized as dots: U+002E (full stop), U+3002 (ideographic full stop), U+FF0E (fullwidth full stop), U+FF61(halfwidth ideographic full stop)"} + ], + "schema": { "format": "idn-hostname" }, + "tests": [ + { + "description": "single dot", + "data": ".", + "valid": false + }, + { + "description": "single ideographic full stop", + "data": "\u3002", + "valid": false + }, + { + "description": "single fullwidth full stop", + "data": "\uff0e", + "valid": false + }, + { + "description": "single halfwidth ideographic full stop", + "data": "\uff61", + "valid": false + }, + { + "description": "dot as label separator", + "data": "a.b", + "valid": true + }, + { + "description": "ideographic full stop as label separator", + "data": "a\u3002b", + "valid": true + }, + { + "description": "fullwidth full stop as label separator", + "data": "a\uff0eb", + "valid": true + }, + { + "description": "halfwidth ideographic full stop as label separator", + "data": "a\uff61b", + "valid": true + }, + { + "description": "leading dot", + "data": ".example", + "valid": false + }, + { + "description": "leading ideographic full stop", + "data": "\u3002example", + "valid": false + }, + { + "description": "leading fullwidth full stop", + "data": "\uff0eexample", + "valid": false + }, + { + "description": "leading halfwidth ideographic full stop", + "data": "\uff61example", + "valid": false + }, + { + "description": "trailing dot", + "data": "example.", + "valid": false + }, + { + "description": "trailing ideographic full stop", + "data": "example\u3002", + "valid": false + }, + { + "description": "trailing fullwidth full stop", + "data": "example\uff0e", + "valid": false + }, + { + "description": "trailing halfwidth ideographic full stop", + "data": "example\uff61", + "valid": false + }, + { + "description": "label too long if separator ignored (full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα.com", + "valid": true + }, + { + "description": "label too long if separator ignored (ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\u3002com", + "valid": true + }, + { + "description": "label too long if separator ignored (fullwidth full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff0ecom", + "valid": true + }, + { + "description": "label too long if separator ignored (halfwidth ideographic full stop)", + "data": "παράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπαράδειγμαπα\uff61com", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json new file mode 100644 index 000000000..d56d62b33 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv4.json @@ -0,0 +1,214 @@ +[ + { + "description": "validation of IP addresses", + "comment": "RFC 2673, Section 3.2: dotted-quad = decbyte \".\" decbyte \".\" decbyte \".\" decbyte. A 'decbyte' (1*3DIGIT) restricts semantic values to 0-255, allows leading zeros, and strictly forbids symbols, alpha/hex, whitespace, and non-ASCII characters.", + "schema": { "format": "ipv4" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IP address", + "data": "192.168.0.1", + "valid": true + }, + { + "description": "an IP address with too many components", + "data": "127.0.0.0.1", + "valid": false + }, + { + "description": "an IP address with out-of-range values", + "data": "256.256.256.256", + "valid": false + }, + { + "description": "an IP address without 4 components", + "data": "127.0", + "valid": false + }, + { + "description": "a 2-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.1", + "valid": false + }, + { + "description": "a 3-part address resolving to a routable IP (inet_aton shorthand)", + "data": "127.0.1", + "valid": false + }, + { + "description": "an IP address as an integer", + "data": "0x7f000001", + "valid": false + }, + { + "description": "an IP address as an integer (decimal)", + "data": "2130706433", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২7.0.0.1", + "valid": false + }, + { + "description": "invalid fullwidth digits (non-ASCII)", + "data": "１９２.１６８.１.１", + "valid": false + }, + { + "description": "invalid mathematical bold digits (non-ASCII)", + "data": "𝟏𝟗𝟐.𝟏𝟔𝟖.𝟏.𝟏", + "valid": false + }, + { + "description": "netmask is not a part of ipv4 address", + "data": "192.168.1.0/24", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " 192.168.0.1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "192.168.0.1 ", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "192.168.0.1\n", + "valid": false + }, + { + "description": "hexadecimal notation is invalid", + "data": "0x7f.0.0.1", + "valid": false + }, + { + "description": "octal notation explicit is invalid", + "data": "0o10.0.0.1", + "valid": false + }, + { + "description": "empty part (double dot) is invalid", + "data": "192.168..1", + "valid": false + }, + { + "description": "leading dot is invalid", + "data": ".192.168.0.1", + "valid": false + }, + { + "description": "trailing dot is invalid", + "data": "192.168.0.1.", + "valid": false + }, + { + "description": "minimum valid IPv4 address", + "data": "0.0.0.0", + "valid": true + }, + { + "description": "maximum valid IPv4 address", + "data": "255.255.255.255", + "valid": true + }, + { + "description": "valid IPv4 with an octet in the 200 to 249 range", + "data": "200.0.0.0", + "valid": true + }, + { + "description": "valid IPv4 with two-digit octets", + "data": "10.20.30.40", + "valid": true + }, + { + "description": "empty string is invalid", + "data": "", + "valid": false + }, + { + "description": "plus sign is invalid", + "data": "+1.2.3.4", + "valid": false + }, + { + "description": "negative sign is invalid", + "data": "-1.2.3.4", + "valid": false + }, + { + "description": "exponential notation is invalid", + "data": "1e2.0.0.1", + "valid": false + }, + { + "description": "alpha characters are invalid", + "data": "192.168.a.1", + "valid": false + }, + { + "description": "internal whitespace is invalid", + "data": "192. 168.0.1", + "valid": false + }, + { + "description": "tab character is invalid", + "data": "192.168.0.1\t", + "valid": false + }, + { + "description": "additional content after an embedded NUL byte", + "data": "192.168.0.1\u0000.evil.com", + "valid": false + }, + { + "description": "with port number is invalid", + "data": "192.168.0.1:80", + "valid": false + }, + { + "description": "an IPv4-mapped IPv6 address is invalid", + "data": "::ffff:192.168.0.1", + "valid": false + }, + { + "description": "single octet out of range in last position", + "data": "192.168.0.256", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json new file mode 100644 index 000000000..7f4674be4 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/ipv6.json @@ -0,0 +1,263 @@ +[ + { + "description": "validation of IPv6 addresses", + "schema": { "format": "ipv6" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IPv6 address", + "data": "::1", + "valid": true + }, + { + "description": "a group with 5 hex digits is invalid", + "data": "12345::", + "valid": false + }, + { + "description": "trailing 4 hex symbols is valid", + "data": "::abef", + "valid": true + }, + { + "description": "trailing 5 hex symbols is invalid", + "data": "::abcef", + "valid": false + }, + { + "description": "an IPv6 address with too many components", + "data": "1:1:1:1:1:1:1:1:1:1:1:1:1:1:1:1", + "valid": false + }, + { + "description": "an IPv6 address containing illegal characters", + "data": "::laptop", + "valid": false + }, + { + "description": "no digits is valid", + "data": "::", + "valid": true + }, + { + "description": "leading colons is valid", + "data": "::42:ff:1", + "valid": true + }, + { + "description": "trailing colons is valid", + "data": "d6::", + "valid": true + }, + { + "description": "missing leading octet is invalid", + "data": ":2:3:4:5:6:7:8", + "valid": false + }, + { + "description": "missing trailing octet is invalid", + "data": "1:2:3:4:5:6:7:", + "valid": false + }, + { + "description": "missing leading octet with omitted octets later", + "data": ":2:3:4::8", + "valid": false + }, + { + "description": "single set of double colons in the middle is valid", + "data": "1:d6::42", + "valid": true + }, + { + "description": "two sets of double colons is invalid", + "data": "1::d6::42", + "valid": false + }, + { + "description": "mixed format with the ipv4 section as decimal octets", + "data": "1::d6:192.168.0.1", + "valid": true + }, + { + "description": "mixed format with double colons between the sections", + "data": "1:2::192.168.0.1", + "valid": true + }, + { + "description": "mixed format with ipv4 section with octet out of range", + "data": "1::2:192.168.256.1", + "valid": false + }, + { + "description": "mixed format with ipv4 section with a hex octet", + "data": "1::2:192.168.ff.1", + "valid": false + }, + { + "description": "mixed format with leading double colons (ipv4-mapped ipv6 address)", + "data": "::ffff:192.168.0.1", + "valid": true + }, + { + "description": "triple colons is invalid", + "data": "1:2:3:4:5:::8", + "valid": false + }, + { + "description": "8 octets", + "data": "1:2:3:4:5:6:7:8", + "valid": true + }, + { + "description": "insufficient octets without double colons", + "data": "1:2:3:4:5:6:7", + "valid": false + }, + { + "description": "no colons is invalid", + "data": "1", + "valid": false + }, + { + "description": "ipv4 is not ipv6", + "data": "127.0.0.1", + "valid": false + }, + { + "description": "ipv4 segment must have 4 octets", + "data": "1:2:3:4:1.2.3", + "valid": false + }, + { + "description": "leading whitespace is invalid", + "data": " ::1", + "valid": false + }, + { + "description": "trailing whitespace is invalid", + "data": "::1 ", + "valid": false + }, + { + "description": "netmask is not a part of ipv6 address", + "data": "fe80::/64", + "valid": false + }, + { + "description": "zone id is not a part of ipv6 address", + "data": "fe80::a%eth1", + "valid": false + }, + { + "description": "a long valid ipv6", + "data": "1000:1000:1000:1000:1000:1000:255.255.255.255", + "valid": true + }, + { + "description": "a long invalid ipv6, below length limit, first", + "data": "100:100:100:100:100:100:255.255.255.255.255", + "valid": false + }, + { + "description": "a long invalid ipv6, below length limit, second", + "data": "100:100:100:100:100:100:100:255.255.255.255", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4)", + "data": "1:2:3:4:5:6:7:৪", + "valid": false + }, + { + "description": "invalid non-ASCII '৪' (a Bengali 4) in the IPv4 portion", + "data": "1:2::192.16৪.0.1", + "valid": false + }, + { + "description": "a bracketed address is invalid", + "data": "[::1]", + "valid": false + }, + { + "description": "a leading zero in the last IPv4 octet is invalid", + "data": "::ffff:192.168.0.01", + "valid": false + }, + { + "description": "uppercase hex digits are valid", + "data": "2001:DB8::1", + "valid": true + }, + { + "description": "leading zeros in every group are valid", + "data": "2001:0db8:0000:0000:0000:0000:0000:0001", + "valid": true + }, + { + "description": "two groups before '::' and four groups after is valid", + "data": "1:2::3:4:5:6", + "valid": true + }, + { + "description": "eight groups alongside '::' is invalid", + "data": "1:2:3:4:5:6:7:8::", + "valid": false + }, + { + "description": "trailing newline is invalid", + "data": "::1\n", + "valid": false + }, + { + "description": "an out-of-range first octet in the embedded IPv4 is invalid", + "data": "::ffff:256.1.1.1", + "valid": false + }, + { + "description": "one group before '::' and five groups after is valid", + "data": "1::2:3:4:5:6", + "valid": true + }, + { + "description": "'::' followed by six groups is valid", + "data": "::1:2:3:4:5:6", + "valid": true + }, + { + "description": "leading '::' before five groups and an embedded IPv4 is valid", + "data": "::1:2:3:4:5:1.2.3.4", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json new file mode 100644 index 000000000..c4b6f1853 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri-reference.json @@ -0,0 +1,113 @@ +[ + { + "description": "validation of IRI References", + "schema": { "format": "iri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid protocol-relative IRI Reference", + "data": "//ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid relative IRI Reference", + "data": "/âππ", + "valid": true + }, + { + "description": "an invalid IRI Reference", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "a valid IRI Reference", + "data": "âππ", + "valid": true + }, + { + "description": "a valid IRI fragment", + "data": "#ƒrägmênt", + "valid": true + }, + { + "description": "an invalid IRI fragment", + "data": "#ƒräg\\mênt", + "valid": false + }, + { + "description": "an embedded IPv4 with a leading zero is invalid", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a valid protocol-relative IRI Reference with a compressed IPv6 host", + "data": "//[2001:db8::1]/p", + "valid": true + }, + { + "description": "a valid relative IRI Reference with a supplementary-plane character", + "data": "/âππ/𐌀", + "valid": true + }, + { + "description": "a query-only IRI Reference with a supplementary private-use char", + "data": "?q=󰀀", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI Reference is invalid", + "data": "/âππ\n", + "valid": false + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "/%", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json new file mode 100644 index 000000000..4025e85d5 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/iri.json @@ -0,0 +1,138 @@ +[ + { + "description": "validation of IRIs", + "schema": { "format": "iri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid IRI with anchor tag", + "data": "http://ƒøø.ßår/?∂éœ=πîx#πîüx", + "valid": true + }, + { + "description": "a valid IRI with anchor tag and parentheses", + "data": "http://ƒøø.com/blah_(wîkïpédiå)_blah#ßité-1", + "valid": true + }, + { + "description": "a valid IRI with URL-encoded stuff", + "data": "http://ƒøø.ßår/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://ƒøø.ßår/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://ƒøø.ßår/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://ƒøø.ßår/%", + "valid": false + }, + { + "description": "a valid IRI with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid IRI based on IPv6", + "data": "http://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", + "valid": true + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + }, + { + "description": "an invalid relative IRI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid IRI", + "data": "\\\\WINDOWS\\filëßåré", + "valid": false + }, + { + "description": "an invalid IRI though valid IRI reference", + "data": "âππ", + "valid": false + }, + { + "description": "an IPv6 host whose embedded IPv4 has a leading zero is invalid", + "data": "http://[::ffff:192.168.0.01]", + "valid": false + }, + { + "description": "a valid IRI with a compressed IPv6 host", + "data": "http://[2001:db8::1]", + "valid": true + }, + { + "description": "a trailing newline after a valid IRI is invalid", + "data": "http://ƒøø.ßår/\n", + "valid": false + }, + { + "description": "a valid IRI with an IPv4 host", + "data": "http://192.168.0.1/p", + "valid": true + }, + { + "description": "a valid IRI with no authority and a rootless path", + "data": "urn:example:resource", + "valid": true + }, + { + "description": "a valid IRI with no authority and an absolute path", + "data": "file:/etc/hosts", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane character in the path", + "data": "http://ƒøø.ßår/𐌀", + "valid": true + }, + { + "description": "a valid IRI with a supplementary-plane private-use char in query", + "data": "http://ƒøø.ßår/?q=󰀀", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json new file mode 100644 index 000000000..134eca586 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/json-pointer.json @@ -0,0 +1,208 @@ +[ + { + "description": "validation of JSON-pointers (JSON String Representation)", + "schema": { "format": "json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid JSON-pointer", + "data": "/foo/bar~0/baz~1/%a", + "valid": true + }, + { + "description": "not a valid JSON-pointer (~ not escaped)", + "data": "/foo/bar~", + "valid": false + }, + { + "description": "valid JSON-pointer with empty segment", + "data": "/foo//bar", + "valid": true + }, + { + "description": "valid JSON-pointer with the last empty segment", + "data": "/foo/bar/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #1", + "data": "", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #2", + "data": "/foo", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #3", + "data": "/foo/0", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #4", + "data": "/", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #5", + "data": "/a~1b", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #6", + "data": "/c%d", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #7", + "data": "/e^f", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #8", + "data": "/g|h", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #9", + "data": "/i\\j", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #10", + "data": "/k\"l", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #11", + "data": "/ ", + "valid": true + }, + { + "description": "valid JSON-pointer as stated in RFC 6901 #12", + "data": "/m~0n", + "valid": true + }, + { + "description": "valid JSON-pointer used adding to the last array position", + "data": "/foo/-", + "valid": true + }, + { + "description": "valid JSON-pointer (- used as object member name)", + "data": "/foo/-/bar", + "valid": true + }, + { + "description": "valid JSON-pointer (multiple escaped characters)", + "data": "/~1~0~0~1~1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #1", + "data": "/~1.1", + "valid": true + }, + { + "description": "valid JSON-pointer (escaped with fraction part) #2", + "data": "/~0.1", + "valid": true + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #1", + "data": "#", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #2", + "data": "#/", + "valid": false + }, + { + "description": "not a valid JSON-pointer (URI Fragment Identifier) #3", + "data": "#a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #1", + "data": "/~0~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (some escaped, but not all) #2", + "data": "/~0/~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #1", + "data": "/~2", + "valid": false + }, + { + "description": "not a valid JSON-pointer (wrong escape character) #2", + "data": "/~-1", + "valid": false + }, + { + "description": "not a valid JSON-pointer (multiple characters not escaped)", + "data": "/~~", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #1", + "data": "a", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #2", + "data": "0", + "valid": false + }, + { + "description": "not a valid JSON-pointer (isn't empty nor starts with /) #3", + "data": "a/a", + "valid": false + }, + { + "description": "valid JSON-pointer (Unicode characters allowed by RFC 6901)", + "data": "/foo/bar/😎", + "valid": true + }, + { + "description": "valid JSON-pointer (control characters allowed after JSON unescaping)", + "data": "/foo\u0000bar\n\tbaz", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json new file mode 100644 index 000000000..ff831fa2e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/regex.json @@ -0,0 +1,100 @@ +[ + { + "description": "validation of regular expressions", + "schema": { "format": "regex" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid regular expression", + "data": "([abc])+\\s+$", + "valid": true + }, + { + "description": "a regular expression with unclosed parens is invalid", + "data": "^(abc]", + "valid": false + }, + { + "description": "a quantifier with nothing to repeat is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns a Quantifier follows something (Term :: Atom Quantifier), so a pattern cannot begin with one", + "data": "*a", + "valid": false + }, + { + "description": "a character class range in reverse order is invalid", + "comment": "https://tc39.es/ecma262/#sec-patterns-static-semantics-early-errors it is a Syntax Error if the first ClassAtom of a range has a greater CharacterValue than the second", + "data": "[z-a]", + "valid": false + }, + { + "description": "a trailing backslash is invalid", + "data": "\\", + "valid": false + }, + { + "description": "an unmatched closing parenthesis is invalid", + "data": "a)", + "valid": false + }, + { + "description": "an unterminated character class is invalid", + "data": "[abc", + "valid": false + }, + { + "description": "an empty regular expression is valid", + "data": "", + "valid": true + }, + { + "description": "alternation is valid", + "data": "a|b", + "valid": true + }, + { + "description": "a bounded quantifier is valid", + "data": "a{2,3}", + "valid": true + }, + { + "description": "a negated range character class is valid", + "data": "[^a-z]", + "valid": true + }, + { + "description": "a lazy quantifier is valid", + "data": "a+?", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json new file mode 100644 index 000000000..59299e7b3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/relative-json-pointer.json @@ -0,0 +1,138 @@ +[ + { + "description": "validation of Relative JSON Pointers (RJP)", + "schema": { "format": "relative-json-pointer" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid upwards RJP", + "data": "1", + "valid": true + }, + { + "description": "a valid downwards RJP", + "data": "0/foo/bar", + "valid": true + }, + { + "description": "a valid up and then down RJP, with array index", + "data": "2/0/baz/1/zip", + "valid": true + }, + { + "description": "a valid RJP taking the member or index name", + "data": "0#", + "valid": true + }, + { + "description": "an invalid RJP that is a valid JSON Pointer", + "data": "/foo/bar", + "valid": false + }, + { + "description": "a json-pointer part that does not start with a slash is invalid", + "data": "1foo", + "valid": false + }, + { + "description": "negative prefix", + "data": "-1/foo/bar", + "valid": false + }, + { + "description": "explicit positive prefix", + "data": "+1/foo/bar", + "valid": false + }, + { + "description": "non-ASCII digit in the prefix is not allowed", + "data": "١/foo", + "valid": false + }, + { + "description": "## is not a valid json-pointer", + "data": "0##", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus json-pointer", + "data": "01/a", + "valid": false + }, + { + "description": "zero cannot be followed by other digits, plus octothorpe", + "data": "01#", + "valid": false + }, + { + "description": "empty string", + "data": "", + "valid": false + }, + { + "description": "multi-digit integer prefix", + "data": "120/foo/bar", + "valid": true + }, + { + "description": "multi-digit prefix with a zero followed by another digit", + "data": "100", + "valid": true + }, + { + "description": "tilde not followed by 0 or 1 in the json-pointer part", + "data": "0/~2", + "valid": false + }, + { + "description": "unescaped tilde at the end of the json-pointer part", + "data": "0/foo/bar~", + "valid": false + }, + { + "description": "octothorpe followed by a json-pointer", + "data": "1#/foo/bar", + "valid": false + }, + { + "description": "empty reference tokens in the json-pointer part", + "data": "0//", + "valid": true + }, + { + "description": "trailing newline after the non-negative integer", + "data": "1\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json new file mode 100644 index 000000000..3182c3ea9 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/time.json @@ -0,0 +1,300 @@ +[ + { + "description": "validation of time strings", + "schema": { "format": "time" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid time string", + "data": "08:30:06Z", + "valid": true + }, + { + "description": "invalid time string with extra leading zeros", + "data": "008:030:006Z", + "valid": false + }, + { + "description": "invalid time string with no leading zero for single digit", + "data": "8:3:6Z", + "valid": false + }, + { + "description": "hour, minute, second must be two digits", + "data": "8:0030:6Z", + "valid": false + }, + { + "description": "a valid time string with leap second, Zulu", + "data": "23:59:60Z", + "valid": true + }, + { + "description": "invalid leap second, Zulu (wrong hour)", + "data": "22:59:60Z", + "valid": false + }, + { + "description": "invalid leap second, Zulu (wrong minute)", + "data": "23:58:60Z", + "valid": false + }, + { + "description": "valid leap second, zero time-offset", + "data": "23:59:60+00:00", + "valid": true + }, + { + "description": "invalid leap second, zero time-offset (wrong hour)", + "data": "22:59:60+00:00", + "valid": false + }, + { + "description": "invalid leap second, zero time-offset (wrong minute)", + "data": "23:58:60+00:00", + "valid": false + }, + { + "description": "valid leap second, positive time-offset", + "data": "01:29:60+01:30", + "valid": true + }, + { + "description": "valid leap second, large positive time-offset", + "data": "23:29:60+23:30", + "valid": true + }, + { + "description": "invalid leap second, positive time-offset (wrong hour)", + "data": "23:59:60+01:00", + "valid": false + }, + { + "description": "invalid leap second, positive time-offset (wrong minute)", + "data": "23:59:60+00:30", + "valid": false + }, + { + "description": "valid leap second, negative time-offset", + "data": "15:59:60-08:00", + "valid": true + }, + { + "description": "valid leap second, large negative time-offset", + "data": "00:29:60-23:30", + "valid": true + }, + { + "description": "invalid leap second, negative time-offset (wrong hour)", + "data": "23:59:60-01:00", + "valid": false + }, + { + "description": "invalid leap second, negative time-offset (wrong minute)", + "data": "23:59:60-00:30", + "valid": false + }, + { + "description": "a valid time string with second fraction", + "data": "23:20:50.52Z", + "valid": true + }, + { + "description": "a valid time string with precise second fraction", + "data": "08:30:06.283185Z", + "valid": true + }, + { + "description": "a valid time string with plus offset", + "data": "08:30:06+00:20", + "valid": true + }, + { + "description": "a valid time string with minus offset", + "data": "08:30:06-08:00", + "valid": true + }, + { + "description": "time with unknown local offset is valid", + "comment": "RFC 3339 section 4.3 (unknown local offset)", + "data": "12:34:56-00:00", + "valid": true + }, + { + "description": "hour, minute in time-offset must be two digits", + "data": "08:30:06-8:000", + "valid": false + }, + { + "description": "a valid time string with case-insensitive Z", + "data": "08:30:06z", + "valid": true + }, + { + "description": "an invalid time string with invalid hour", + "data": "24:00:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid minute", + "data": "00:60:00Z", + "valid": false + }, + { + "description": "an invalid time string with invalid second", + "data": "00:00:61Z", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset hour", + "data": "01:02:03+24:00", + "valid": false + }, + { + "description": "an invalid time string with invalid time numoffset minute", + "data": "01:02:03+00:60", + "valid": false + }, + { + "description": "an invalid time string with invalid time with both Z and numoffset", + "data": "01:02:03Z+00:30", + "valid": false + }, + { + "description": "an invalid offset indicator", + "data": "08:30:06 PST", + "valid": false + }, + { + "description": "only RFC3339 not all of ISO 8601 are valid", + "data": "01:01:01,1111", + "valid": false + }, + { + "description": "no time offset", + "data": "12:00:00", + "valid": false + }, + { + "description": "no time offset with second fraction", + "data": "12:00:00.52", + "valid": false + }, + { + "description": "invalid non-ASCII '২' (a Bengali 2)", + "data": "1২:00:00Z", + "valid": false + }, + { + "description": "offset not starting with plus or minus", + "data": "08:30:06#00:20", + "valid": false + }, + { + "description": "contains letters", + "data": "ab:cd:ef", + "valid": false + }, + { + "description": "an invalid time string in date-time format", + "data": "2020-11-28T23:55:45Z", + "valid": false + }, + { + "description": "a numeric time-offset requires the colon separator", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+0130", + "valid": false + }, + { + "description": "a numeric time-offset requires the minutes component", + "comment": "RFC 3339 section 5.6: time-numoffset = (\"+\" / \"-\") time-hour \":\" time-minute", + "data": "08:30:06+01", + "valid": false + }, + { + "description": "a trailing newline after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits a line feed", + "data": "08:30:06Z\n", + "valid": false + }, + { + "description": "hour 24 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-hour = 2DIGIT ; 00-23, and section 5.7 only allows 00 to 23", + "data": "24:59:00+01:00", + "valid": false + }, + { + "description": "minute 60 is invalid with a numeric time-offset", + "comment": "RFC 3339 section 5.6: time-minute = 2DIGIT ; 00-59", + "data": "23:60:00+00:01", + "valid": false + }, + { + "description": "a valid time string with a 15 digit second fraction", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT, with no upper bound on the number of digits", + "data": "00:59:59.999999999999999Z", + "valid": true + }, + { + "description": "a second fraction with no digits", + "comment": "RFC 3339 section 5.6: time-secfrac = \".\" 1*DIGIT requires at least one digit", + "data": "08:30:06.Z", + "valid": false + }, + { + "description": "partial-time requires the seconds component", + "comment": "RFC 3339 section 5.6: partial-time = time-hour \":\" time-minute \":\" time-second [time-secfrac]", + "data": "12:00Z", + "valid": false + }, + { + "description": "a comma decimal separator with a time-offset", + "comment": "RFC 3339 section 5.6: the decimal separator in time-secfrac is \".\" - the ISO 8601 comma is not part of this profile", + "data": "08:30:06,5Z", + "valid": false + }, + { + "description": "leading whitespace before a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": " 08:30:06Z", + "valid": false + }, + { + "description": "trailing whitespace after a valid time", + "comment": "RFC 3339 section 5.6: no production of full-time emits whitespace", + "data": "08:30:06Z ", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json new file mode 100644 index 000000000..12339ae57 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/unknown.json @@ -0,0 +1,43 @@ +[ + { + "description": "unknown format", + "schema": { "format": "unknown" }, + "tests": [ + { + "description": "unknown formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "unknown formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "unknown formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "unknown formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "unknown formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "unknown formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "unknown formats ignore strings", + "data": "string", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json new file mode 100644 index 000000000..3741bad01 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-reference.json @@ -0,0 +1,179 @@ +[ + { + "description": "validation of URI References", + "schema": { "format": "uri-reference" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URI", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid relative URI Reference", + "data": "/abc", + "valid": true + }, + { + "description": "an invalid URI Reference", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "a valid URI Reference", + "data": "abc", + "valid": true + }, + { + "description": "a valid URI fragment", + "data": "#fragment", + "valid": true + }, + { + "description": "an invalid URI fragment", + "data": "#frag\\ment", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "an empty URI reference", + "comment": "RFC 3986, Section 4.2: relative-part includes path-empty, which is zero characters.", + "data": "", + "valid": true + }, + { + "description": "a query-only reference", + "comment": "RFC 3986, Section 4.2: relative-ref = relative-part [ \"?\" query ] [ \"#\" fragment ], so a query with an empty path is a reference.", + "data": "?query=1", + "valid": true + }, + { + "description": "a network-path reference with an empty authority", + "comment": "RFC 3986, Section 3.2.2: host = IP-literal / IPv4address / reg-name, and reg-name may be empty.", + "data": "//", + "valid": true + }, + { + "description": "an incomplete percent-encoding", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%zz", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "comment": "RFC 3986, Section 2.1: pct-encoded = \"%\" HEXDIG HEXDIG.", + "data": "/%", + "valid": false + }, + { + "description": "a double quote in a path", + "comment": "RFC 3986, Appendix A: pchar = unreserved / pct-encoded / sub-delims / \":\" / \"@\", and the double quote is in none of them.", + "data": "/a\"b", + "valid": false + }, + { + "description": "square brackets outside an authority", + "comment": "RFC 3986, Section 3.2.2: square brackets are reserved for an IP-literal inside the authority. path-absolute is built from pchar, which excludes them.", + "data": "/[::1]", + "valid": false + }, + { + "description": "a non-numeric port in a network-path reference", + "comment": "RFC 3986, Section 3.2.3: port = *DIGIT. reg-name admits no colon, so the text after the colon can only be a port.", + "data": "//example.com:abc/p", + "valid": false + }, + { + "description": "more than one at-sign in the authority", + "comment": "RFC 3986, Section 3.2: authority = [ userinfo \"@\" ] host [ \":\" port ]. Neither userinfo nor host admits a literal at-sign.", + "data": "//a@b@example.com/", + "valid": false + }, + { + "description": "a leading zero in the IPv4 part of an IPv6 literal", + "comment": "RFC 3986, Section 3.2.2: inside an IP-literal the dotted quad must be an IPv4address, and dec-octet has no leading-zero alternative. There is no reg-name fallback inside brackets.", + "data": "//[::ffff:192.168.0.01]/p", + "valid": false + }, + { + "description": "a colon in the first segment of a relative-path reference", + "comment": "RFC 3986, Section 4.2: relative-part uses path-noscheme, whose first segment is segment-nz-nc - a non-zero-length segment without any colon. A scheme must start with ALPHA, so 1:b cannot be read as a URI either.", + "data": "1:b", + "valid": false + }, + { + "description": "a colon in the first segment preceded by a dot-segment", + "comment": "RFC 3986, Section 4.2: \"Such a segment must be preceded by a dot-segment (e.g., \\\"./this:that\\\") to make a relative-path reference.\"", + "data": "./this:that", + "valid": true + }, + { + "description": "a trailing line feed", + "comment": "RFC 3986, Appendix A: no production admits %x0A.", + "data": "/p\n", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json new file mode 100644 index 000000000..f0aaa52f7 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri-template.json @@ -0,0 +1,248 @@ +[ + { + "description": "format: uri-template", + "schema": { "format": "uri-template" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an invalid uri-template", + "data": "http://example.com/dictionary/{term:1}/{term", + "valid": false + }, + { + "description": "a valid uri-template without variables", + "data": "http://example.com/dictionary", + "valid": true + }, + { + "description": "a valid relative uri-template", + "data": "dictionary/{term:1}/{term}", + "valid": true + }, + { + "description": "an empty expression is invalid", + "data": "{}", + "valid": false + }, + { + "description": "an empty varspec inside the variable list is invalid", + "data": "{a,,b}", + "valid": false + }, + { + "description": "a zero prefix max-length is invalid", + "data": "{v:0}", + "valid": false + }, + { + "description": "a five-digit prefix max-length is invalid", + "data": "{v:10000}", + "valid": false + }, + { + "description": "an empty string is a valid uri-template", + "data": "", + "valid": true + }, + { + "description": "an unmatched closing brace is invalid", + "data": "foo}bar", + "valid": false + }, + { + "description": "a percent-encoded triplet in a literal is valid", + "data": "a%41b", + "valid": true + }, + { + "description": "a literal with a lone percent sign is invalid", + "data": "a%", + "valid": false + }, + { + "description": "a literal with an incomplete percent-encoded triplet is invalid", + "data": "a%4", + "valid": false + }, + { + "description": "a literal with non-hex percent encoding is invalid", + "data": "a%GG", + "valid": false + }, + { + "description": "a supplementary plane character in a literal is valid", + "data": "a\ud83d\ude00b", + "valid": true + }, + { + "description": "a private-use character in a literal is valid", + "data": "a\ue000b", + "valid": true + }, + { + "description": "a supplementary private-use character in a literal is valid", + "data": "a\udb80\udc00b", + "valid": true + }, + { + "description": "reserved expansion operator", + "data": "{+var}", + "valid": true + }, + { + "description": "fragment expansion operator", + "data": "{#var}", + "valid": true + }, + { + "description": "label expansion operator", + "data": "{.var}", + "valid": true + }, + { + "description": "path segment expansion operator", + "data": "{/var}", + "valid": true + }, + { + "description": "path style parameter expansion operator", + "data": "{;var}", + "valid": true + }, + { + "description": "form style query expansion operator", + "data": "{?var}", + "valid": true + }, + { + "description": "form style query continuation operator", + "data": "{&var}", + "valid": true + }, + { + "description": "the explode modifier is valid", + "data": "{var*}", + "valid": true + }, + { + "description": "combined prefix and explode modifiers are invalid", + "data": "{var:1*}", + "valid": false + }, + { + "description": "a variable list with more than one varspec is valid", + "data": "{x,y}", + "valid": true + }, + { + "description": "a dotted variable name is valid", + "data": "{a.b}", + "valid": true + }, + { + "description": "a trailing dot in a variable name is invalid", + "data": "{a.}", + "valid": false + }, + { + "description": "a double dot in a variable name is invalid", + "data": "{a..b}", + "valid": false + }, + { + "description": "a variable name may start with a percent-encoded triplet", + "data": "{%41}", + "valid": true + }, + { + "description": "a variable name with a lone percent sign is invalid", + "data": "{%}", + "valid": false + }, + { + "description": "a variable name with an incomplete percent-encoded triplet is invalid", + "data": "{%4}", + "valid": false + }, + { + "description": "a variable name with non-hex percent encoding is invalid", + "data": "{%GG}", + "valid": false + }, + { + "description": "a four-digit prefix max-length is valid", + "data": "{v:1000}", + "valid": true + }, + { + "description": "a leading zero in the prefix max-length is invalid", + "data": "{v:01}", + "valid": false + }, + { + "description": "a space in a literal is invalid", + "data": "a b", + "valid": false + }, + { + "description": "a delete character in a literal is invalid", + "data": "a\u007fb", + "valid": false + }, + { + "description": "a trailing comma in the variable list is invalid", + "data": "{a,}", + "valid": false + }, + { + "description": "a default value in a varspec is invalid", + "data": "{var=def}", + "valid": false + }, + { + "description": "an operator in place of the first varspec is invalid", + "data": "{,+var}", + "valid": false + }, + { + "description": "an apostrophe in a literal is valid", + "data": "a'b", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json new file mode 100644 index 000000000..b9d080027 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/format/uri.json @@ -0,0 +1,250 @@ +[ + { + "description": "validation of URIs", + "schema": { "format": "uri" }, + "tests": [ + { + "description": "all string formats ignore integers", + "data": 12, + "valid": true + }, + { + "description": "all string formats ignore floats", + "data": 13.7, + "valid": true + }, + { + "description": "all string formats ignore objects", + "data": {}, + "valid": true + }, + { + "description": "all string formats ignore arrays", + "data": [], + "valid": true + }, + { + "description": "all string formats ignore booleans", + "data": false, + "valid": true + }, + { + "description": "all string formats ignore nulls", + "data": null, + "valid": true + }, + { + "description": "a valid URL with anchor tag", + "data": "http://foo.bar/?baz=qux#quux", + "valid": true + }, + { + "description": "a valid URL with anchor tag and parentheses", + "data": "http://foo.com/blah_(wikipedia)_blah#cite-1", + "valid": true + }, + { + "description": "a valid URL with URL-encoded stuff", + "data": "http://foo.bar/?q=Test%20URL-encoded%20stuff", + "valid": true + }, + { + "description": "a valid puny-coded URL ", + "data": "http://xn--nw2a.xn--j6w193g/", + "valid": true + }, + { + "description": "a valid URL with many special characters", + "data": "http://-.~_!$&'()*+,;=:%40:80%2f::::::@example.com", + "valid": true + }, + { + "description": "a valid URL based on IPv4", + "data": "http://223.255.255.254", + "valid": true + }, + { + "description": "a valid URL with ftp scheme", + "data": "ftp://ftp.is.co.za/rfc/rfc1808.txt", + "valid": true + }, + { + "description": "a valid URL for a simple text file", + "data": "http://www.ietf.org/rfc/rfc2396.txt", + "valid": true + }, + { + "description": "a valid URL", + "data": "ldap://[2001:db8::7]/c=GB?objectClass?one", + "valid": true + }, + { + "description": "a valid mailto URI", + "data": "mailto:John.Doe@example.com", + "valid": true + }, + { + "description": "a valid newsgroup URI", + "data": "news:comp.infosystems.www.servers.unix", + "valid": true + }, + { + "description": "a valid tel URI", + "data": "tel:+1-816-555-1212", + "valid": true + }, + { + "description": "a valid URN", + "data": "urn:oasis:names:specification:docbook:dtd:xml:4.1.2", + "valid": true + }, + { + "description": "an invalid protocol-relative URI Reference", + "data": "//foo.bar/?baz=qux#quux", + "valid": false + }, + { + "description": "an invalid relative URI Reference", + "data": "/abc", + "valid": false + }, + { + "description": "an invalid URI", + "data": "\\\\WINDOWS\\fileshare", + "valid": false + }, + { + "description": "an invalid URI though valid URI reference", + "data": "abc", + "valid": false + }, + { + "description": "an invalid URI with spaces", + "data": "http:// shouldfail.com", + "valid": false + }, + { + "description": "an invalid URI with spaces and missing scheme", + "data": ":// should fail", + "valid": false + }, + { + "description": "an invalid URI with comma in scheme", + "data": "bar,baz:foo", + "valid": false + }, + { + "description": "invalid userinfo", + "data": "https://[@example.org/test.txt", + "valid": false + }, + { + "description": "unescaped non US-ASCII characters", + "data": "https://example.org/foobar®.txt", + "valid": false + }, + { + "description": "invalid backslash character", + "data": "https://example.org/foobar\\.txt", + "valid": false + }, + { + "description": "invalid \" character", + "data": "https://example.org/foobar\".txt", + "valid": false + }, + { + "description": "invalid <> characters", + "data": "https://example.org/foobar<>.txt", + "valid": false + }, + { + "description": "invalid {} characters", + "data": "https://example.org/foobar{}.txt", + "valid": false + }, + { + "description": "invalid ^ character", + "data": "https://example.org/foobar^.txt", + "valid": false + }, + { + "description": "invalid ` character", + "data": "https://example.org/foobar`.txt", + "valid": false + }, + { + "description": "invalid SPACE character", + "data": "https://example.org/foo bar.txt", + "valid": false + }, + { + "description": "invalid | character", + "data": "https://example.org/foobar|.txt", + "valid": false + }, + { + "description": "URI with leading-zero IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: If an IP literal fails strict IPv4address parsing, it falls back to reg-name. A string of digits and dots is valid under unreserved characters. JSON Schema format asserts syntax, not scheme-specific DNS semantics.", + "data": "http://087.10.0.1/", + "valid": true + }, + { + "description": "URI with out-of-bounds IPv4 is structurally valid as a reg-name", + "comment": "RFC 3986, Section 3.2.2: Fallback to reg-name allows digits and dots.", + "data": "http://999.999.999.999/", + "valid": true + }, + { + "description": "invalid percent-encoding with non-hex digits", + "data": "http://example.com/%6G", + "valid": false + }, + { + "description": "incomplete percent-encoding triplet", + "data": "http://example.com/%A", + "valid": false + }, + { + "description": "lone percent sign is invalid", + "data": "http://example.com/%", + "valid": false + }, + { + "description": "scheme must start with a letter", + "data": "1http://example.com", + "valid": false + }, + { + "description": "invalid character in scheme", + "data": "ht_tp://example.com", + "valid": false + }, + { + "description": "non-numeric port is invalid", + "data": "http://example.com:abc/path", + "valid": false + }, + { + "description": "leading zero in an embedded IPv4 address is invalid", + "data": "http://[::ffff:01.2.3.4]", + "valid": false + }, + { + "description": "square brackets are not allowed in a path segment", + "data": "http:/[::1]", + "valid": false + }, + { + "description": "a trailing newline after a valid URI is invalid", + "data": "http://foo.bar/\n", + "valid": false + }, + { + "description": "an IPv6 address without enclosing brackets is invalid", + "data": "http://2001:0db8:85a3:0000:0000:8a2e:0370:7334", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json new file mode 100644 index 000000000..6be81b8da --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/id.json @@ -0,0 +1,114 @@ +[ + { + "description": "id inside an enum is not a real identifier", + "comment": "the implementation must not be confused by an id buried in the enum", + "schema": { + "definitions": { + "id_in_enum": { + "enum": [ + { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + } + ] + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "string" + }, + "zzz_id_in_const": { + "const": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + } + } + }, + "anyOf": [ + { "$ref": "#/definitions/id_in_enum" }, + { "$ref": "https://localhost:1234/id/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "exact match to enum, and type matches", + "data": { + "$id": "https://localhost:1234/id/my_identifier.json", + "type": "null" + }, + "valid": true + }, + { + "description": "match $ref to id", + "data": "a string to match #/definitions/id_in_enum", + "valid": true + }, + { + "description": "no match on enum or $ref to id", + "data": 1, + "valid": false + } + ] + }, + { + "description": "non-schema object containing a plain-name $id property", + "schema": { + "definitions": { + "const_not_anchor": { + "const": { + "$id": "#not_a_real_anchor" + } + } + }, + "if": { + "const": "skip not_a_real_anchor" + }, + "then": true, + "else" : { + "$ref": "#/definitions/const_not_anchor" + } + }, + "tests": [ + { + "description": "skip traversing definition for a valid result", + "data": "skip not_a_real_anchor", + "valid": true + }, + { + "description": "const at const_not_anchor does not match", + "data": 1, + "valid": false + } + ] + }, + { + "description": "non-schema object containing an $id property", + "schema": { + "definitions": { + "const_not_id": { + "const": { + "$id": "not_a_real_id" + } + } + }, + "if": { + "const": "skip not_a_real_id" + }, + "then": true, + "else" : { + "$ref": "#/definitions/const_not_id" + } + }, + "tests": [ + { + "description": "skip traversing definition for a valid result", + "data": "skip not_a_real_id", + "valid": true + }, + { + "description": "const at const_not_id does not match", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json new file mode 100644 index 000000000..dd67af2b2 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/non-bmp-regex.json @@ -0,0 +1,82 @@ +[ + { + "description": "Proper UTF-16 surrogate pair handling: pattern", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { "pattern": "^🐲*$" }, + "tests": [ + { + "description": "matches empty", + "data": "", + "valid": true + }, + { + "description": "matches single", + "data": "🐲", + "valid": true + }, + { + "description": "matches two", + "data": "🐲🐲", + "valid": true + }, + { + "description": "doesn't match one", + "data": "🐉", + "valid": false + }, + { + "description": "doesn't match two", + "data": "🐉🐉", + "valid": false + }, + { + "description": "doesn't match one ASCII", + "data": "D", + "valid": false + }, + { + "description": "doesn't match two ASCII", + "data": "DD", + "valid": false + } + ] + }, + { + "description": "Proper UTF-16 surrogate pair handling: patternProperties", + "comment": "Optional because .Net doesn't correctly handle 32-bit Unicode characters", + "schema": { + "patternProperties": { + "^🐲*$": { + "type": "integer" + } + } + }, + "tests": [ + { + "description": "matches empty", + "data": { "": 1 }, + "valid": true + }, + { + "description": "matches single", + "data": { "🐲": 1 }, + "valid": true + }, + { + "description": "matches two", + "data": { "🐲🐲": 1 }, + "valid": true + }, + { + "description": "doesn't match one", + "data": { "🐲": "hello" }, + "valid": false + }, + { + "description": "doesn't match two", + "data": { "🐲🐲": "hello" }, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json new file mode 100644 index 000000000..1f58d97e3 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/optional/unknownKeyword.json @@ -0,0 +1,56 @@ +[ + { + "description": "$id inside an unknown keyword is not a real identifier", + "comment": "the implementation must not be confused by an $id in locations we do not know how to parse", + "schema": { + "definitions": { + "id_in_unknown0": { + "not": { + "array_of_schemas": [ + { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "null" + } + ] + } + }, + "real_id_in_schema": { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "string" + }, + "id_in_unknown1": { + "not": { + "object_of_schemas": { + "foo": { + "$id": "https://localhost:1234/unknownKeyword/my_identifier.json", + "type": "integer" + } + } + } + } + }, + "anyOf": [ + { "$ref": "#/definitions/id_in_unknown0" }, + { "$ref": "#/definitions/id_in_unknown1" }, + { "$ref": "https://localhost:1234/unknownKeyword/my_identifier.json" } + ] + }, + "tests": [ + { + "description": "type matches second anyOf, which has a real schema in it", + "data": "a string", + "valid": true + }, + { + "description": "type matches non-schema in first anyOf", + "data": null, + "valid": false + }, + { + "description": "type matches non-schema in third anyOf", + "data": 1, + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json new file mode 100644 index 000000000..92db0f971 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/pattern.json @@ -0,0 +1,59 @@ +[ + { + "description": "pattern validation", + "schema": {"pattern": "^a*$"}, + "tests": [ + { + "description": "a matching pattern is valid", + "data": "aaa", + "valid": true + }, + { + "description": "a non-matching pattern is invalid", + "data": "abc", + "valid": false + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + }, + { + "description": "ignores integers", + "data": 123, + "valid": true + }, + { + "description": "ignores floats", + "data": 1.0, + "valid": true + }, + { + "description": "ignores objects", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + } + ] + }, + { + "description": "pattern is not anchored", + "schema": {"pattern": "a+"}, + "tests": [ + { + "description": "matches a substring", + "data": "xxaayy", + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json new file mode 100644 index 000000000..c276e6479 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/patternProperties.json @@ -0,0 +1,171 @@ +[ + { + "description": + "patternProperties validates properties matching a regex", + "schema": { + "patternProperties": { + "f.*o": {"type": "integer"} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "multiple valid matches is valid", + "data": {"foo": 1, "foooooo" : 2}, + "valid": true + }, + { + "description": "a single invalid match is invalid", + "data": {"foo": "bar", "fooooo": 2}, + "valid": false + }, + { + "description": "multiple invalid matches is invalid", + "data": {"foo": "bar", "foooooo" : "baz"}, + "valid": false + }, + { + "description": "ignores arrays", + "data": ["foo"], + "valid": true + }, + { + "description": "ignores strings", + "data": "foo", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": "multiple simultaneous patternProperties are validated", + "schema": { + "patternProperties": { + "a*": {"type": "integer"}, + "aaa*": {"maximum": 20} + } + }, + "tests": [ + { + "description": "a single valid match is valid", + "data": {"a": 21}, + "valid": true + }, + { + "description": "a simultaneous match is valid", + "data": {"aaaa": 18}, + "valid": true + }, + { + "description": "multiple matches is valid", + "data": {"a": 21, "aaaa": 18}, + "valid": true + }, + { + "description": "an invalid due to one is invalid", + "data": {"a": "bar"}, + "valid": false + }, + { + "description": "an invalid due to the other is invalid", + "data": {"aaaa": 31}, + "valid": false + }, + { + "description": "an invalid due to both is invalid", + "data": {"aaa": "foo", "aaaa": 31}, + "valid": false + } + ] + }, + { + "description": "regexes are not anchored by default and are case sensitive", + "schema": { + "patternProperties": { + "[0-9]{2,}": { "type": "boolean" }, + "X_": { "type": "string" } + } + }, + "tests": [ + { + "description": "non recognized members are ignored", + "data": { "answer 1": "42" }, + "valid": true + }, + { + "description": "recognized members are accounted for", + "data": { "a31b": null }, + "valid": false + }, + { + "description": "regexes are case sensitive", + "data": { "a_x_3": 3 }, + "valid": true + }, + { + "description": "regexes are case sensitive, 2", + "data": { "a_X_3": 3 }, + "valid": false + } + ] + }, + { + "description": "patternProperties with boolean schemas", + "schema": { + "patternProperties": { + "f.*": true, + "b.*": false + } + }, + "tests": [ + { + "description": "object with property matching schema true is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property matching schema false is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "object with both properties is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + }, + { + "description": "object with a property matching both true and false is invalid", + "data": {"foobar":1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "patternProperties with null valued instance properties", + "schema": { + "patternProperties": { + "^.*bar$": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foobar": null}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json new file mode 100644 index 000000000..5b971ca0e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/properties.json @@ -0,0 +1,236 @@ +[ + { + "description": "object properties validation", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "both properties present and valid is valid", + "data": {"foo": 1, "bar": "baz"}, + "valid": true + }, + { + "description": "one property invalid is invalid", + "data": {"foo": 1, "bar": {}}, + "valid": false + }, + { + "description": "both properties invalid is invalid", + "data": {"foo": [], "bar": {}}, + "valid": false + }, + { + "description": "doesn't invalidate other properties", + "data": {"quux": []}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + } + ] + }, + { + "description": + "properties, patternProperties, additionalProperties interaction", + "schema": { + "properties": { + "foo": {"type": "array", "maxItems": 3}, + "bar": {"type": "array"} + }, + "patternProperties": {"f.o": {"minItems": 2}}, + "additionalProperties": {"type": "integer"} + }, + "tests": [ + { + "description": "property validates property", + "data": {"foo": [1, 2]}, + "valid": true + }, + { + "description": "property invalidates property", + "data": {"foo": [1, 2, 3, 4]}, + "valid": false + }, + { + "description": "patternProperty invalidates property", + "data": {"foo": []}, + "valid": false + }, + { + "description": "patternProperty validates nonproperty", + "data": {"fxo": [1, 2]}, + "valid": true + }, + { + "description": "patternProperty invalidates nonproperty", + "data": {"fxo": []}, + "valid": false + }, + { + "description": "additionalProperty ignores property", + "data": {"bar": []}, + "valid": true + }, + { + "description": "additionalProperty validates others", + "data": {"quux": 3}, + "valid": true + }, + { + "description": "additionalProperty invalidates others", + "data": {"quux": "foo"}, + "valid": false + } + ] + }, + { + "description": "properties with boolean schema", + "schema": { + "properties": { + "foo": true, + "bar": false + } + }, + "tests": [ + { + "description": "no property present is valid", + "data": {}, + "valid": true + }, + { + "description": "only 'true' property present is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "only 'false' property present is invalid", + "data": {"bar": 2}, + "valid": false + }, + { + "description": "both properties present is invalid", + "data": {"foo": 1, "bar": 2}, + "valid": false + } + ] + }, + { + "description": "properties with escaped characters", + "schema": { + "properties": { + "foo\nbar": {"type": "number"}, + "foo\"bar": {"type": "number"}, + "foo\\bar": {"type": "number"}, + "foo\rbar": {"type": "number"}, + "foo\tbar": {"type": "number"}, + "foo\fbar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with all numbers is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1", + "foo\\bar": "1", + "foo\rbar": "1", + "foo\tbar": "1", + "foo\fbar": "1" + }, + "valid": false + } + ] + }, + { + "description": "properties with null valued instance properties", + "schema": { + "properties": { + "foo": {"type": "null"} + } + }, + "tests": [ + { + "description": "allows null values", + "data": {"foo": null}, + "valid": true + } + ] + }, + { + "description": "properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { + "properties": { + "__proto__": {"type": "number"}, + "toString": { + "properties": { "length": { "type": "string" } } + }, + "constructor": {"type": "number"} + } + }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": true + }, + { + "description": "__proto__ not valid", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString not valid", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor not valid", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present and valid", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json new file mode 100644 index 000000000..b86564310 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/propertyNames.json @@ -0,0 +1,164 @@ +[ + { + "description": "propertyNames validation", + "schema": { + "propertyNames": {"maxLength": 3} + }, + "tests": [ + { + "description": "all property names valid", + "data": { + "f": {}, + "foo": {} + }, + "valid": true + }, + { + "description": "some property names invalid", + "data": { + "foo": {}, + "foobar": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + }, + { + "description": "ignores arrays", + "data": [1, 2, 3, 4], + "valid": true + }, + { + "description": "ignores strings", + "data": "foobar", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores booleans", + "data": true, + "valid": true + } + ] + }, + { + "description": "propertyNames validation with pattern", + "schema": { + "propertyNames": { "pattern": "^a+$" } + }, + "tests": [ + { + "description": "matching property names valid", + "data": { + "a": {}, + "aa": {}, + "aaa": {} + }, + "valid": true + }, + { + "description": "non-matching property name is invalid", + "data": { + "aaA": {} + }, + "valid": false + }, + { + "description": "object without properties is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema true", + "schema": {"propertyNames": true}, + "tests": [ + { + "description": "object with any properties is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with boolean schema false", + "schema": {"propertyNames": false}, + "tests": [ + { + "description": "object with any properties is invalid", + "data": {"foo": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with const", + "schema": {"propertyNames": {"const": "foo"}}, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + }, + { + "description": "propertyNames with enum", + "schema": {"propertyNames": {"enum": ["foo", "bar"]}}, + "tests": [ + { + "description": "object with property foo is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "object with property foo and bar is valid", + "data": {"foo": 1, "bar": 1}, + "valid": true + }, + { + "description": "object with any other property is invalid", + "data": {"baz": 1}, + "valid": false + }, + { + "description": "empty object is valid", + "data": {}, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json new file mode 100644 index 000000000..82e1e1672 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/ref.json @@ -0,0 +1,1043 @@ +[ + { + "description": "root pointer ref", + "schema": { + "properties": { + "foo": {"$ref": "#"} + }, + "additionalProperties": false + }, + "tests": [ + { + "description": "match", + "data": {"foo": false}, + "valid": true + }, + { + "description": "recursive match", + "data": {"foo": {"foo": false}}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": false}, + "valid": false + }, + { + "description": "recursive mismatch", + "data": {"foo": {"bar": false}}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to object", + "schema": { + "properties": { + "foo": {"type": "integer"}, + "bar": {"$ref": "#/properties/foo"} + } + }, + "tests": [ + { + "description": "match", + "data": {"bar": 3}, + "valid": true + }, + { + "description": "mismatch", + "data": {"bar": true}, + "valid": false + } + ] + }, + { + "description": "relative pointer ref to array", + "schema": { + "items": [ + {"type": "integer"}, + {"$ref": "#/items/0"} + ] + }, + "tests": [ + { + "description": "match array", + "data": [1, 2], + "valid": true + }, + { + "description": "mismatch array", + "data": [1, "foo"], + "valid": false + } + ] + }, + { + "description": "escaped pointer ref", + "schema": { + "definitions": { + "tilde~field": {"type": "integer"}, + "slash/field": {"type": "integer"}, + "percent%field": {"type": "integer"} + }, + "properties": { + "tilde": {"$ref": "#/definitions/tilde~0field"}, + "slash": {"$ref": "#/definitions/slash~1field"}, + "percent": {"$ref": "#/definitions/percent%25field"} + } + }, + "tests": [ + { + "description": "slash invalid", + "data": {"slash": "aoeu"}, + "valid": false + }, + { + "description": "tilde invalid", + "data": {"tilde": "aoeu"}, + "valid": false + }, + { + "description": "percent invalid", + "data": {"percent": "aoeu"}, + "valid": false + }, + { + "description": "slash valid", + "data": {"slash": 123}, + "valid": true + }, + { + "description": "tilde valid", + "data": {"tilde": 123}, + "valid": true + }, + { + "description": "percent valid", + "data": {"percent": 123}, + "valid": true + } + ] + }, + { + "description": "nested refs", + "schema": { + "definitions": { + "a": {"type": "integer"}, + "b": {"$ref": "#/definitions/a"}, + "c": {"$ref": "#/definitions/b"} + }, + "allOf": [{ "$ref": "#/definitions/c" }] + }, + "tests": [ + { + "description": "nested ref valid", + "data": 5, + "valid": true + }, + { + "description": "nested ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref overrides any sibling keywords", + "schema": { + "definitions": { + "reffed": { + "type": "array" + } + }, + "properties": { + "foo": { + "$ref": "#/definitions/reffed", + "maxItems": 2 + } + } + }, + "tests": [ + { + "description": "ref valid", + "data": { "foo": [] }, + "valid": true + }, + { + "description": "ref valid, maxItems ignored", + "data": { "foo": [ 1, 2, 3] }, + "valid": true + }, + { + "description": "ref invalid", + "data": { "foo": "string" }, + "valid": false + } + ] + }, + { + "description": "$ref prevents a sibling $id from changing the base uri", + "schema": { + "$id": "http://localhost:1234/sibling_id/base/", + "definitions": { + "foo": { + "$id": "http://localhost:1234/sibling_id/foo.json", + "type": "string" + }, + "base_foo": { + "$comment": "this canonical uri is http://localhost:1234/sibling_id/base/foo.json", + "$id": "foo.json", + "type": "number" + } + }, + "allOf": [ + { + "$comment": "$ref resolves to http://localhost:1234/sibling_id/base/foo.json, not http://localhost:1234/sibling_id/foo.json", + "$id": "http://localhost:1234/sibling_id/", + "$ref": "foo.json" + } + ] + }, + "tests": [ + { + "description": "$ref resolves to /definitions/base_foo, data does not validate", + "data": "a", + "valid": false + }, + { + "description": "$ref resolves to /definitions/base_foo, data validates", + "data": 1, + "valid": true + } + ] + }, + { + "description": "remote ref, containing refs itself", + "schema": {"$ref": "http://json-schema.org/draft-07/schema#"}, + "tests": [ + { + "description": "remote ref valid", + "data": {"minLength": 1}, + "valid": true + }, + { + "description": "remote ref invalid", + "data": {"minLength": -1}, + "valid": false + } + ] + }, + { + "description": "property named $ref that is not a reference", + "schema": { + "properties": { + "$ref": {"type": "string"} + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "property named $ref, containing an actual $ref", + "schema": { + "properties": { + "$ref": {"$ref": "#/definitions/is-string"} + }, + "definitions": { + "is-string": { + "type": "string" + } + } + }, + "tests": [ + { + "description": "property named $ref valid", + "data": {"$ref": "a"}, + "valid": true + }, + { + "description": "property named $ref invalid", + "data": {"$ref": 2}, + "valid": false + } + ] + }, + { + "description": "$ref to boolean schema true", + "schema": { + "allOf": [{ "$ref": "#/definitions/bool" }], + "definitions": { + "bool": true + } + }, + "tests": [ + { + "description": "any value is valid", + "data": "foo", + "valid": true + } + ] + }, + { + "description": "$ref to boolean schema false", + "schema": { + "allOf": [{ "$ref": "#/definitions/bool" }], + "definitions": { + "bool": false + } + }, + "tests": [ + { + "description": "any value is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "Recursive references between schemas", + "schema": { + "$id": "http://localhost:1234/tree", + "description": "tree of nodes", + "type": "object", + "properties": { + "meta": {"type": "string"}, + "nodes": { + "type": "array", + "items": {"$ref": "node"} + } + }, + "required": ["meta", "nodes"], + "definitions": { + "node": { + "$id": "http://localhost:1234/node", + "description": "node", + "type": "object", + "properties": { + "value": {"type": "number"}, + "subtree": {"$ref": "tree"} + }, + "required": ["value"] + } + } + }, + "tests": [ + { + "description": "valid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 1.1}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": true + }, + { + "description": "invalid tree", + "data": { + "meta": "root", + "nodes": [ + { + "value": 1, + "subtree": { + "meta": "child", + "nodes": [ + {"value": "string is invalid"}, + {"value": 1.2} + ] + } + }, + { + "value": 2, + "subtree": { + "meta": "child", + "nodes": [ + {"value": 2.1}, + {"value": 2.2} + ] + } + } + ] + }, + "valid": false + } + ] + }, + { + "description": "refs with quote", + "schema": { + "properties": { + "foo\"bar": {"$ref": "#/definitions/foo%22bar"} + }, + "definitions": { + "foo\"bar": {"type": "number"} + } + }, + "tests": [ + { + "description": "object with numbers is valid", + "data": { + "foo\"bar": 1 + }, + "valid": true + }, + { + "description": "object with strings is invalid", + "data": { + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "Location-independent identifier", + "schema": { + "allOf": [{ + "$ref": "#foo" + }], + "definitions": { + "A": { + "$id": "#foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Reference an anchor with a non-relative URI", + "schema": { + "$id": "https://example.com/schema-with-anchor", + "allOf": [{ + "$ref": "https://example.com/schema-with-anchor#foo" + }], + "definitions": { + "A": { + "$id": "#foo", + "type": "integer" + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "Location-independent identifier with base URI change in subschema", + "schema": { + "$id": "http://localhost:1234/root", + "allOf": [{ + "$ref": "http://localhost:1234/nested.json#foo" + }], + "definitions": { + "A": { + "$id": "nested.json", + "definitions": { + "B": { + "$id": "#foo", + "type": "integer" + } + } + } + } + }, + "tests": [ + { + "data": 1, + "description": "match", + "valid": true + }, + { + "data": "a", + "description": "mismatch", + "valid": false + } + ] + }, + { + "description": "naive replacement of $ref with its destination is not correct", + "schema": { + "definitions": { + "a_string": { "type": "string" } + }, + "enum": [ + { "$ref": "#/definitions/a_string" } + ] + }, + "tests": [ + { + "description": "do not evaluate the $ref inside the enum, matching any string", + "data": "this is a string", + "valid": false + }, + { + "description": "do not evaluate the $ref inside the enum, definition exact match", + "data": { "type": "string" }, + "valid": false + }, + { + "description": "match the enum exactly", + "data": { "$ref": "#/definitions/a_string" }, + "valid": true + } + ] + }, + { + "description": "refs with relative uris and defs", + "schema": { + "$id": "http://example.com/schema-relative-uri-defs1.json", + "properties": { + "foo": { + "$id": "schema-relative-uri-defs2.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] + } + }, + "allOf": [ { "$ref": "schema-relative-uri-defs2.json" } ] + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "relative refs with absolute uris and defs", + "schema": { + "$id": "http://example.com/schema-refs-absolute-uris-defs1.json", + "properties": { + "foo": { + "$id": "http://example.com/schema-refs-absolute-uris-defs2.json", + "definitions": { + "inner": { + "properties": { + "bar": { "type": "string" } + } + } + }, + "allOf": [ { "$ref": "#/definitions/inner" } ] + } + }, + "allOf": [ { "$ref": "schema-refs-absolute-uris-defs2.json" } ] + }, + "tests": [ + { + "description": "invalid on inner field", + "data": { + "foo": { + "bar": 1 + }, + "bar": "a" + }, + "valid": false + }, + { + "description": "invalid on outer field", + "data": { + "foo": { + "bar": "a" + }, + "bar": 1 + }, + "valid": false + }, + { + "description": "valid on both fields", + "data": { + "foo": { + "bar": "a" + }, + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "$id must be resolved against nearest parent, not just immediate parent", + "schema": { + "$id": "http://example.com/a.json", + "definitions": { + "x": { + "$id": "http://example.com/b/c.json", + "not": { + "definitions": { + "y": { + "$id": "d.json", + "type": "number" + } + } + } + } + }, + "allOf": [ + { + "$ref": "http://example.com/b/d.json" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "simple URN base URI with $ref via the URN", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$id": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed", + "minimum": 30, + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ffff-ffff-4321feebdaed"} + } + }, + "tests": [ + { + "description": "valid under the URN IDed schema", + "data": {"foo": 37}, + "valid": true + }, + { + "description": "invalid under the URN IDed schema", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "simple URN base URI with JSON pointer", + "schema": { + "$comment": "URIs do not have to have HTTP(s) schemes", + "$id": "urn:uuid:deadbeef-1234-00ff-ff00-4321feebdaed", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with NSS", + "schema": { + "$comment": "RFC 8141 §2.2", + "$id": "urn:example:1/406/47452/2", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with r-component", + "schema": { + "$comment": "RFC 8141 §2.3.1", + "$id": "urn:example:foo-bar-baz-qux?+CCResolve:cc=uk", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with q-component", + "schema": { + "$comment": "RFC 8141 §2.3.2", + "$id": "urn:example:weather?=op=map&lat=39.56&lon=-104.85&datetime=1969-07-21T02:56:15Z", + "properties": { + "foo": {"$ref": "#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and JSON pointer ref", + "schema": { + "$id": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-0000-0000-4321feebdaed#/definitions/bar"} + }, + "definitions": { + "bar": {"type": "string"} + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "URN base URI with URN and anchor ref", + "schema": { + "$id": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed", + "properties": { + "foo": {"$ref": "urn:uuid:deadbeef-1234-ff00-00ff-4321feebdaed#something"} + }, + "definitions": { + "bar": { + "$id": "#something", + "type": "string" + } + } + }, + "tests": [ + { + "description": "a string is valid", + "data": {"foo": "bar"}, + "valid": true + }, + { + "description": "a non-string is invalid", + "data": {"foo": 12}, + "valid": false + } + ] + }, + { + "description": "ref to if", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/if"}, + { + "if": { + "$id": "http://example.com/ref/if", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to then", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/then"}, + { + "then": { + "$id": "http://example.com/ref/then", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref to else", + "schema": { + "allOf": [ + {"$ref": "http://example.com/ref/else"}, + { + "else": { + "$id": "http://example.com/ref/else", + "type": "integer" + } + } + ] + }, + "tests": [ + { + "description": "a non-integer is invalid due to the $ref", + "data": "foo", + "valid": false + }, + { + "description": "an integer is valid", + "data": 12, + "valid": true + } + ] + }, + { + "description": "ref with absolute-path-reference", + "schema": { + "$id": "http://example.com/ref/absref.json", + "definitions": { + "a": { + "$id": "http://example.com/ref/absref/foobar.json", + "type": "number" + }, + "b": { + "$id": "http://example.com/absref/foobar.json", + "type": "string" + } + }, + "allOf": [ + { "$ref": "/absref/foobar.json" } + ] + }, + "tests": [ + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "an integer is invalid", + "data": 12, + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - *nix", + "schema": { + "$id": "file:///folder/file.json", + "definitions": { + "foo": { + "type": "number" + } + }, + "allOf": [ + { + "$ref": "#/definitions/foo" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "$id with file URI still resolves pointers - windows", + "schema": { + "$id": "file:///c:/folder/file.json", + "definitions": { + "foo": { + "type": "number" + } + }, + "allOf": [ + { + "$ref": "#/definitions/foo" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "empty tokens in $ref json-pointer", + "schema": { + "definitions": { + "": { + "definitions": { + "": { "type": "number" } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions//definitions/" + } + ] + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json new file mode 100644 index 000000000..450787af6 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/refRemote.json @@ -0,0 +1,257 @@ +[ + { + "description": "remote ref", + "schema": {"$ref": "http://localhost:1234/integer.json"}, + "tests": [ + { + "description": "remote ref valid", + "data": 1, + "valid": true + }, + { + "description": "remote ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "fragment within remote ref", + "schema": {"$ref": "http://localhost:1234/draft7/subSchemas.json#/definitions/integer"}, + "tests": [ + { + "description": "remote fragment valid", + "data": 1, + "valid": true + }, + { + "description": "remote fragment invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "ref within remote ref", + "schema": { + "$ref": "http://localhost:1234/draft7/subSchemas.json#/definitions/refToInteger" + }, + "tests": [ + { + "description": "ref within ref valid", + "data": 1, + "valid": true + }, + { + "description": "ref within ref invalid", + "data": "a", + "valid": false + } + ] + }, + { + "description": "base URI change", + "schema": { + "$id": "http://localhost:1234/", + "items": { + "$id": "baseUriChange/", + "items": {"$ref": "folderInteger.json"} + } + }, + "tests": [ + { + "description": "base URI change ref valid", + "data": [[1]], + "valid": true + }, + { + "description": "base URI change ref invalid", + "data": [["a"]], + "valid": false + } + ] + }, + { + "description": "base URI change - change folder", + "schema": { + "$id": "http://localhost:1234/scope_change_defs1.json", + "type" : "object", + "properties": { + "list": {"$ref": "#/definitions/baz"} + }, + "definitions": { + "baz": { + "$id": "baseUriChangeFolder/", + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "base URI change - change folder in subschema", + "schema": { + "$id": "http://localhost:1234/scope_change_defs2.json", + "type" : "object", + "properties": { + "list": {"$ref": "#/definitions/baz/definitions/bar"} + }, + "definitions": { + "baz": { + "$id": "baseUriChangeFolderInSubschema/", + "definitions": { + "bar": { + "type": "array", + "items": {"$ref": "folderInteger.json"} + } + } + } + } + }, + "tests": [ + { + "description": "number is valid", + "data": {"list": [1]}, + "valid": true + }, + { + "description": "string is invalid", + "data": {"list": ["a"]}, + "valid": false + } + ] + }, + { + "description": "root ref in remote ref", + "schema": { + "$id": "http://localhost:1234/object", + "type": "object", + "properties": { + "name": {"$ref": "draft7/name.json#/definitions/orNull"} + } + }, + "tests": [ + { + "description": "string is valid", + "data": { + "name": "foo" + }, + "valid": true + }, + { + "description": "null is valid", + "data": { + "name": null + }, + "valid": true + }, + { + "description": "object is invalid", + "data": { + "name": { + "name": null + } + }, + "valid": false + } + ] + }, + { + "description": "remote ref with ref to definitions", + "schema": { + "$id": "http://localhost:1234/schema-remote-ref-ref-defs1.json", + "allOf": [ + { "$ref": "draft7/ref-and-definitions.json" } + ] + }, + "tests": [ + { + "description": "invalid", + "data": { + "bar": 1 + }, + "valid": false + }, + { + "description": "valid", + "data": { + "bar": "a" + }, + "valid": true + } + ] + }, + { + "description": "Location-independent identifier in remote ref", + "schema": { + "$ref": "http://localhost:1234/draft7/locationIndependentIdentifier.json#/definitions/refToInteger" + }, + "tests": [ + { + "description": "integer is valid", + "data": 1, + "valid": true + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + }, + { + "description": "retrieved nested refs resolve relative to their URI not $id", + "schema": { + "$id": "http://localhost:1234/some-id", + "properties": { + "name": {"$ref": "nested/foo-ref-string.json"} + } + }, + "tests": [ + { + "description": "number is invalid", + "data": { + "name": {"foo": 1} + }, + "valid": false + }, + { + "description": "string is valid", + "data": { + "name": {"foo": "a"} + }, + "valid": true + } + ] + }, + { + "description": "$ref to $ref finds location-independent $id", + "schema": { + "$ref": "http://localhost:1234/draft7/detached-ref.json#/definitions/foo" + }, + "tests": [ + { + "description": "number is valid", + "data": 1, + "valid": true + }, + { + "description": "non-number is invalid", + "data": "a", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json new file mode 100644 index 000000000..f98c64af8 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/required.json @@ -0,0 +1,161 @@ +[ + { + "description": "required validation", + "schema": { + "properties": { + "foo": {}, + "bar": {} + }, + "required": ["foo"] + }, + "tests": [ + { + "description": "present required property is valid", + "data": {"foo": 1}, + "valid": true + }, + { + "description": "non-present required property is invalid", + "data": {"bar": 1}, + "valid": false + }, + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores strings", + "data": "", + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "ignores null", + "data": null, + "valid": true + }, + { + "description": "ignores boolean", + "data": true, + "valid": true + } + ] + }, + { + "description": "required default validation", + "schema": { + "properties": { + "foo": {} + } + }, + "tests": [ + { + "description": "not required by default", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with empty array", + "schema": { + "properties": { + "foo": {} + }, + "required": [] + }, + "tests": [ + { + "description": "property not required", + "data": {}, + "valid": true + } + ] + }, + { + "description": "required with escaped characters", + "schema": { + "required": [ + "foo\nbar", + "foo\"bar", + "foo\\bar", + "foo\rbar", + "foo\tbar", + "foo\fbar" + ] + }, + "tests": [ + { + "description": "object with all properties present is valid", + "data": { + "foo\nbar": 1, + "foo\"bar": 1, + "foo\\bar": 1, + "foo\rbar": 1, + "foo\tbar": 1, + "foo\fbar": 1 + }, + "valid": true + }, + { + "description": "object with some properties missing is invalid", + "data": { + "foo\nbar": "1", + "foo\"bar": "1" + }, + "valid": false + } + ] + }, + { + "description": "required properties whose names are Javascript object property names", + "comment": "Ensure JS implementations don't universally consider e.g. __proto__ to always be present in an object.", + "schema": { "required": ["__proto__", "toString", "constructor"] }, + "tests": [ + { + "description": "ignores arrays", + "data": [], + "valid": true + }, + { + "description": "ignores other non-objects", + "data": 12, + "valid": true + }, + { + "description": "none of the properties mentioned", + "data": {}, + "valid": false + }, + { + "description": "__proto__ present", + "data": { "__proto__": "foo" }, + "valid": false + }, + { + "description": "toString present", + "data": { "toString": { "length": 37 } }, + "valid": false + }, + { + "description": "constructor present", + "data": { "constructor": { "length": 37 } }, + "valid": false + }, + { + "description": "all present", + "data": { + "__proto__": 12, + "toString": { "length": "foo" }, + "constructor": 37 + }, + "valid": true + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json new file mode 100644 index 000000000..830464702 --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/type.json @@ -0,0 +1,474 @@ +[ + { + "description": "integer type matches integers", + "schema": {"type": "integer"}, + "tests": [ + { + "description": "an integer is an integer", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is an integer", + "data": 1.0, + "valid": true + }, + { + "description": "a float is not an integer", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an integer", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not an integer, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not an integer", + "data": {}, + "valid": false + }, + { + "description": "an array is not an integer", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an integer", + "data": true, + "valid": false + }, + { + "description": "null is not an integer", + "data": null, + "valid": false + } + ] + }, + { + "description": "number type matches numbers", + "schema": {"type": "number"}, + "tests": [ + { + "description": "an integer is a number", + "data": 1, + "valid": true + }, + { + "description": "a float with zero fractional part is a number (and an integer)", + "data": 1.0, + "valid": true + }, + { + "description": "a float is a number", + "data": 1.1, + "valid": true + }, + { + "description": "a string is not a number", + "data": "foo", + "valid": false + }, + { + "description": "a string is still not a number, even if it looks like one", + "data": "1", + "valid": false + }, + { + "description": "an object is not a number", + "data": {}, + "valid": false + }, + { + "description": "an array is not a number", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a number", + "data": true, + "valid": false + }, + { + "description": "null is not a number", + "data": null, + "valid": false + } + ] + }, + { + "description": "string type matches strings", + "schema": {"type": "string"}, + "tests": [ + { + "description": "1 is not a string", + "data": 1, + "valid": false + }, + { + "description": "a float is not a string", + "data": 1.1, + "valid": false + }, + { + "description": "a string is a string", + "data": "foo", + "valid": true + }, + { + "description": "a string is still a string, even if it looks like a number", + "data": "1", + "valid": true + }, + { + "description": "an empty string is still a string", + "data": "", + "valid": true + }, + { + "description": "an object is not a string", + "data": {}, + "valid": false + }, + { + "description": "an array is not a string", + "data": [], + "valid": false + }, + { + "description": "a boolean is not a string", + "data": true, + "valid": false + }, + { + "description": "null is not a string", + "data": null, + "valid": false + } + ] + }, + { + "description": "object type matches objects", + "schema": {"type": "object"}, + "tests": [ + { + "description": "an integer is not an object", + "data": 1, + "valid": false + }, + { + "description": "a float is not an object", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an object", + "data": "foo", + "valid": false + }, + { + "description": "an object is an object", + "data": {}, + "valid": true + }, + { + "description": "an array is not an object", + "data": [], + "valid": false + }, + { + "description": "a boolean is not an object", + "data": true, + "valid": false + }, + { + "description": "null is not an object", + "data": null, + "valid": false + } + ] + }, + { + "description": "array type matches arrays", + "schema": {"type": "array"}, + "tests": [ + { + "description": "an integer is not an array", + "data": 1, + "valid": false + }, + { + "description": "a float is not an array", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not an array", + "data": "foo", + "valid": false + }, + { + "description": "an object is not an array", + "data": {}, + "valid": false + }, + { + "description": "an array is an array", + "data": [], + "valid": true + }, + { + "description": "a boolean is not an array", + "data": true, + "valid": false + }, + { + "description": "null is not an array", + "data": null, + "valid": false + } + ] + }, + { + "description": "boolean type matches booleans", + "schema": {"type": "boolean"}, + "tests": [ + { + "description": "an integer is not a boolean", + "data": 1, + "valid": false + }, + { + "description": "zero is not a boolean", + "data": 0, + "valid": false + }, + { + "description": "a float is not a boolean", + "data": 1.1, + "valid": false + }, + { + "description": "a string is not a boolean", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not a boolean", + "data": "", + "valid": false + }, + { + "description": "an object is not a boolean", + "data": {}, + "valid": false + }, + { + "description": "an array is not a boolean", + "data": [], + "valid": false + }, + { + "description": "true is a boolean", + "data": true, + "valid": true + }, + { + "description": "false is a boolean", + "data": false, + "valid": true + }, + { + "description": "null is not a boolean", + "data": null, + "valid": false + } + ] + }, + { + "description": "null type matches only the null object", + "schema": {"type": "null"}, + "tests": [ + { + "description": "an integer is not null", + "data": 1, + "valid": false + }, + { + "description": "a float is not null", + "data": 1.1, + "valid": false + }, + { + "description": "zero is not null", + "data": 0, + "valid": false + }, + { + "description": "a string is not null", + "data": "foo", + "valid": false + }, + { + "description": "an empty string is not null", + "data": "", + "valid": false + }, + { + "description": "an object is not null", + "data": {}, + "valid": false + }, + { + "description": "an array is not null", + "data": [], + "valid": false + }, + { + "description": "true is not null", + "data": true, + "valid": false + }, + { + "description": "false is not null", + "data": false, + "valid": false + }, + { + "description": "null is null", + "data": null, + "valid": true + } + ] + }, + { + "description": "multiple types can be specified in an array", + "schema": {"type": ["integer", "string"]}, + "tests": [ + { + "description": "an integer is valid", + "data": 1, + "valid": true + }, + { + "description": "a string is valid", + "data": "foo", + "valid": true + }, + { + "description": "a float is invalid", + "data": 1.1, + "valid": false + }, + { + "description": "an object is invalid", + "data": {}, + "valid": false + }, + { + "description": "an array is invalid", + "data": [], + "valid": false + }, + { + "description": "a boolean is invalid", + "data": true, + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type as array with one item", + "schema": { + "type": ["string"] + }, + "tests": [ + { + "description": "string is valid", + "data": "foo", + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + } + ] + }, + { + "description": "type: array or object", + "schema": { + "type": ["array", "object"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + }, + { + "description": "null is invalid", + "data": null, + "valid": false + } + ] + }, + { + "description": "type: array, object or null", + "schema": { + "type": ["array", "object", "null"] + }, + "tests": [ + { + "description": "array is valid", + "data": [1,2,3], + "valid": true + }, + { + "description": "object is valid", + "data": {"foo": 123}, + "valid": true + }, + { + "description": "null is valid", + "data": null, + "valid": true + }, + { + "description": "number is invalid", + "data": 123, + "valid": false + }, + { + "description": "string is invalid", + "data": "foo", + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json new file mode 100644 index 000000000..a1da0096e --- /dev/null +++ b/packages/schema/test/fixtures/json-schema-test-suite/tests/draft7/uniqueItems.json @@ -0,0 +1,389 @@ +[ + { + "description": "uniqueItems validation", + "schema": {"uniqueItems": true}, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is invalid", + "data": [1, 1], + "valid": false + }, + { + "description": "non-unique array of more than two integers is invalid", + "data": [1, 2, 1], + "valid": false + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": false + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of strings is valid", + "data": ["foo", "bar", "baz"], + "valid": true + }, + { + "description": "non-unique array of strings is invalid", + "data": ["foo", "bar", "foo"], + "valid": false + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is invalid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": false + }, + { + "description": "property order of array of objects is ignored", + "data": [{"foo": "bar", "bar": "foo"}, {"bar": "foo", "foo": "bar"}], + "valid": false + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is invalid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": false + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is invalid", + "data": [["foo"], ["foo"]], + "valid": false + }, + { + "description": "non-unique array of more than two arrays is invalid", + "data": [["foo"], ["bar"], ["foo"]], + "valid": false + }, + { + "description": "[1] and [true] are unique", + "data": [[1], [true]], + "valid": true + }, + { + "description": "[0] and [false] are unique", + "data": [[0], [false]], + "valid": true + }, + { + "description": "nested [1] and [true] are unique", + "data": [[[1], "foo"], [[true], "foo"]], + "valid": true + }, + { + "description": "nested [0] and [false] are unique", + "data": [[[0], "foo"], [[false], "foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1, "{}"], + "valid": true + }, + { + "description": "non-unique heterogeneous types are invalid", + "data": [{}, [1], true, null, {}, 1], + "valid": false + }, + { + "description": "different objects are unique", + "data": [{"a": 1, "b": 2}, {"a": 2, "b": 1}], + "valid": true + }, + { + "description": "objects are non-unique despite key order", + "data": [{"a": 1, "b": 2}, {"b": 2, "a": 1}], + "valid": false + }, + { + "description": "{\"a\": false} and {\"a\": 0} are unique", + "data": [{"a": false}, {"a": 0}], + "valid": true + }, + { + "description": "{\"a\": true} and {\"a\": 1} are unique", + "data": [{"a": true}, {"a": 1}], + "valid": true + } + ] + }, + { + "description": "uniqueItems with an array of items", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is not valid", + "data": [false, true, "foo", "foo"], + "valid": false + }, + { + "description": "non-unique array extended from [true, false] is not valid", + "data": [true, false, "foo", "foo"], + "valid": false + } + ] + }, + { + "description": "uniqueItems with an array of items and additionalItems=false", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": true, + "additionalItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is not valid", + "data": [false, false], + "valid": false + }, + { + "description": "[true, true] from items array is not valid", + "data": [true, true], + "valid": false + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + }, + { + "description": "uniqueItems=false validation", + "schema": { "uniqueItems": false }, + "tests": [ + { + "description": "unique array of integers is valid", + "data": [1, 2], + "valid": true + }, + { + "description": "non-unique array of integers is valid", + "data": [1, 1], + "valid": true + }, + { + "description": "numbers are unique if mathematically unequal", + "data": [1.0, 1.00, 1], + "valid": true + }, + { + "description": "false is not equal to zero", + "data": [0, false], + "valid": true + }, + { + "description": "true is not equal to one", + "data": [1, true], + "valid": true + }, + { + "description": "unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "baz"}], + "valid": true + }, + { + "description": "non-unique array of objects is valid", + "data": [{"foo": "bar"}, {"foo": "bar"}], + "valid": true + }, + { + "description": "unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : false}}} + ], + "valid": true + }, + { + "description": "non-unique array of nested objects is valid", + "data": [ + {"foo": {"bar" : {"baz" : true}}}, + {"foo": {"bar" : {"baz" : true}}} + ], + "valid": true + }, + { + "description": "unique array of arrays is valid", + "data": [["foo"], ["bar"]], + "valid": true + }, + { + "description": "non-unique array of arrays is valid", + "data": [["foo"], ["foo"]], + "valid": true + }, + { + "description": "unique heterogeneous types are valid", + "data": [{}, [1], true, null, 1], + "valid": true + }, + { + "description": "non-unique heterogeneous types are valid", + "data": [{}, [1], true, null, {}, 1], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "unique array extended from [false, true] is valid", + "data": [false, true, "foo", "bar"], + "valid": true + }, + { + "description": "unique array extended from [true, false] is valid", + "data": [true, false, "foo", "bar"], + "valid": true + }, + { + "description": "non-unique array extended from [false, true] is valid", + "data": [false, true, "foo", "foo"], + "valid": true + }, + { + "description": "non-unique array extended from [true, false] is valid", + "data": [true, false, "foo", "foo"], + "valid": true + } + ] + }, + { + "description": "uniqueItems=false with an array of items and additionalItems=false", + "schema": { + "items": [{"type": "boolean"}, {"type": "boolean"}], + "uniqueItems": false, + "additionalItems": false + }, + "tests": [ + { + "description": "[false, true] from items array is valid", + "data": [false, true], + "valid": true + }, + { + "description": "[true, false] from items array is valid", + "data": [true, false], + "valid": true + }, + { + "description": "[false, false] from items array is valid", + "data": [false, false], + "valid": true + }, + { + "description": "[true, true] from items array is valid", + "data": [true, true], + "valid": true + }, + { + "description": "extra items are invalid even if unique", + "data": [false, true, null], + "valid": false + } + ] + } +] diff --git a/packages/schema/test/fixtures/meta-schemas/README.md b/packages/schema/test/fixtures/meta-schemas/README.md new file mode 100644 index 000000000..46b120d97 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/README.md @@ -0,0 +1,10 @@ +# Meta-schemas + +The official meta-schemas some suite tests reference, unmodified: + +- `draft2020-12-schema.json`: https://json-schema.org/draft/2020-12/schema +- `draft2020-12-meta-*.json`: https://json-schema.org/draft/2020-12/meta/{core,applicator,unevaluated,validation,meta-data,format-annotation,content} +- `draft-07-schema.json`: http://json-schema.org/draft-07/schema + +Published by the JSON Schema organization (https://github.com/json-schema-org/json-schema-spec), licensed +AFL-3.0 or BSD-3-Clause. diff --git a/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json b/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json new file mode 100644 index 000000000..fb92c7f75 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft-07-schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "http://json-schema.org/draft-07/schema#", + "title": "Core schema meta-schema", + "definitions": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#" } + }, + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "allOf": [ + { "$ref": "#/definitions/nonNegativeInteger" }, + { "default": 0 } + ] + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + }, + "type": ["object", "boolean"], + "properties": { + "$id": { + "type": "string", + "format": "uri-reference" + }, + "$schema": { + "type": "string", + "format": "uri" + }, + "$ref": { + "type": "string", + "format": "uri-reference" + }, + "$comment": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/definitions/nonNegativeInteger" }, + "minLength": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "additionalItems": { "$ref": "#" }, + "items": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/schemaArray" } + ], + "default": true + }, + "maxItems": { "$ref": "#/definitions/nonNegativeInteger" }, + "minItems": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "contains": { "$ref": "#" }, + "maxProperties": { "$ref": "#/definitions/nonNegativeInteger" }, + "minProperties": { "$ref": "#/definitions/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/definitions/stringArray" }, + "additionalProperties": { "$ref": "#" }, + "definitions": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "properties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$ref": "#" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependencies": { + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$ref": "#" }, + { "$ref": "#/definitions/stringArray" } + ] + } + }, + "propertyNames": { "$ref": "#" }, + "const": true, + "enum": { + "type": "array", + "items": true, + "minItems": 1, + "uniqueItems": true + }, + "type": { + "anyOf": [ + { "$ref": "#/definitions/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/definitions/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "format": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentEncoding": { "type": "string" }, + "if": { "$ref": "#" }, + "then": { "$ref": "#" }, + "else": { "$ref": "#" }, + "allOf": { "$ref": "#/definitions/schemaArray" }, + "anyOf": { "$ref": "#/definitions/schemaArray" }, + "oneOf": { "$ref": "#/definitions/schemaArray" }, + "not": { "$ref": "#" } + }, + "default": true +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json new file mode 100644 index 000000000..f4775974a --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-applicator.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/applicator", + "$dynamicAnchor": "meta", + + "title": "Applicator vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "prefixItems": { "$ref": "#/$defs/schemaArray" }, + "items": { "$dynamicRef": "#meta" }, + "contains": { "$dynamicRef": "#meta" }, + "additionalProperties": { "$dynamicRef": "#meta" }, + "properties": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "default": {} + }, + "patternProperties": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "propertyNames": { "format": "regex" }, + "default": {} + }, + "dependentSchemas": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "default": {} + }, + "propertyNames": { "$dynamicRef": "#meta" }, + "if": { "$dynamicRef": "#meta" }, + "then": { "$dynamicRef": "#meta" }, + "else": { "$dynamicRef": "#meta" }, + "allOf": { "$ref": "#/$defs/schemaArray" }, + "anyOf": { "$ref": "#/$defs/schemaArray" }, + "oneOf": { "$ref": "#/$defs/schemaArray" }, + "not": { "$dynamicRef": "#meta" } + }, + "$defs": { + "schemaArray": { + "type": "array", + "minItems": 1, + "items": { "$dynamicRef": "#meta" } + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json new file mode 100644 index 000000000..76e3760d2 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-content.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/content", + "$dynamicAnchor": "meta", + + "title": "Content vocabulary meta-schema", + + "type": ["object", "boolean"], + "properties": { + "contentEncoding": { "type": "string" }, + "contentMediaType": { "type": "string" }, + "contentSchema": { "$dynamicRef": "#meta" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json new file mode 100644 index 000000000..691862289 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-core.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/core", + "$dynamicAnchor": "meta", + + "title": "Core vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "$id": { + "$ref": "#/$defs/uriReferenceString", + "$comment": "Non-empty fragments not allowed.", + "pattern": "^[^#]*#?$" + }, + "$schema": { "$ref": "#/$defs/uriString" }, + "$ref": { "$ref": "#/$defs/uriReferenceString" }, + "$anchor": { "$ref": "#/$defs/anchorString" }, + "$dynamicRef": { "$ref": "#/$defs/uriReferenceString" }, + "$dynamicAnchor": { "$ref": "#/$defs/anchorString" }, + "$vocabulary": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/uriString" }, + "additionalProperties": { + "type": "boolean" + } + }, + "$comment": { + "type": "string" + }, + "$defs": { + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" } + } + }, + "$defs": { + "anchorString": { + "type": "string", + "pattern": "^[A-Za-z_][-A-Za-z0-9._]*$" + }, + "uriString": { + "type": "string", + "format": "uri" + }, + "uriReferenceString": { + "type": "string", + "format": "uri-reference" + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json new file mode 100644 index 000000000..3479e6695 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-format-annotation.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/format-annotation", + "$dynamicAnchor": "meta", + + "title": "Format vocabulary meta-schema for annotation results", + "type": ["object", "boolean"], + "properties": { + "format": { "type": "string" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json new file mode 100644 index 000000000..4049ab21b --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-meta-data.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/meta-data", + "$dynamicAnchor": "meta", + + "title": "Meta-data vocabulary meta-schema", + + "type": ["object", "boolean"], + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "default": true, + "deprecated": { + "type": "boolean", + "default": false + }, + "readOnly": { + "type": "boolean", + "default": false + }, + "writeOnly": { + "type": "boolean", + "default": false + }, + "examples": { + "type": "array", + "items": true + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json new file mode 100644 index 000000000..93779e54e --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-unevaluated.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/unevaluated", + "$dynamicAnchor": "meta", + + "title": "Unevaluated applicator vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "unevaluatedItems": { "$dynamicRef": "#meta" }, + "unevaluatedProperties": { "$dynamicRef": "#meta" } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json new file mode 100644 index 000000000..ebb75db77 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-meta-validation.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/meta/validation", + "$dynamicAnchor": "meta", + + "title": "Validation vocabulary meta-schema", + "type": ["object", "boolean"], + "properties": { + "type": { + "anyOf": [ + { "$ref": "#/$defs/simpleTypes" }, + { + "type": "array", + "items": { "$ref": "#/$defs/simpleTypes" }, + "minItems": 1, + "uniqueItems": true + } + ] + }, + "const": true, + "enum": { + "type": "array", + "items": true + }, + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0 + }, + "maximum": { + "type": "number" + }, + "exclusiveMaximum": { + "type": "number" + }, + "minimum": { + "type": "number" + }, + "exclusiveMinimum": { + "type": "number" + }, + "maxLength": { "$ref": "#/$defs/nonNegativeInteger" }, + "minLength": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "pattern": { + "type": "string", + "format": "regex" + }, + "maxItems": { "$ref": "#/$defs/nonNegativeInteger" }, + "minItems": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "uniqueItems": { + "type": "boolean", + "default": false + }, + "maxContains": { "$ref": "#/$defs/nonNegativeInteger" }, + "minContains": { + "$ref": "#/$defs/nonNegativeInteger", + "default": 1 + }, + "maxProperties": { "$ref": "#/$defs/nonNegativeInteger" }, + "minProperties": { "$ref": "#/$defs/nonNegativeIntegerDefault0" }, + "required": { "$ref": "#/$defs/stringArray" }, + "dependentRequired": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/stringArray" + } + } + }, + "$defs": { + "nonNegativeInteger": { + "type": "integer", + "minimum": 0 + }, + "nonNegativeIntegerDefault0": { + "$ref": "#/$defs/nonNegativeInteger", + "default": 0 + }, + "simpleTypes": { + "enum": [ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string" + ] + }, + "stringArray": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "default": [] + } + } +} diff --git a/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json b/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json new file mode 100644 index 000000000..d5e2d31c3 --- /dev/null +++ b/packages/schema/test/fixtures/meta-schemas/draft2020-12-schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://json-schema.org/draft/2020-12/schema", + "$vocabulary": { + "https://json-schema.org/draft/2020-12/vocab/core": true, + "https://json-schema.org/draft/2020-12/vocab/applicator": true, + "https://json-schema.org/draft/2020-12/vocab/unevaluated": true, + "https://json-schema.org/draft/2020-12/vocab/validation": true, + "https://json-schema.org/draft/2020-12/vocab/meta-data": true, + "https://json-schema.org/draft/2020-12/vocab/format-annotation": true, + "https://json-schema.org/draft/2020-12/vocab/content": true + }, + "$dynamicAnchor": "meta", + + "title": "Core and Validation specifications meta-schema", + "allOf": [ + {"$ref": "meta/core"}, + {"$ref": "meta/applicator"}, + {"$ref": "meta/unevaluated"}, + {"$ref": "meta/validation"}, + {"$ref": "meta/meta-data"}, + {"$ref": "meta/format-annotation"}, + {"$ref": "meta/content"} + ], + "type": ["object", "boolean"], + "$comment": "This meta-schema also defines keywords that have appeared in previous drafts in order to prevent incompatible extensions as they remain in common use.", + "properties": { + "definitions": { + "$comment": "\"definitions\" has been replaced by \"$defs\".", + "type": "object", + "additionalProperties": { "$dynamicRef": "#meta" }, + "deprecated": true, + "default": {} + }, + "dependencies": { + "$comment": "\"dependencies\" has been split and replaced by \"dependentSchemas\" and \"dependentRequired\" in order to serve their differing semantics.", + "type": "object", + "additionalProperties": { + "anyOf": [ + { "$dynamicRef": "#meta" }, + { "$ref": "meta/validation#/$defs/stringArray" } + ] + }, + "deprecated": true, + "default": {} + }, + "$recursiveAnchor": { + "$comment": "\"$recursiveAnchor\" has been replaced by \"$dynamicAnchor\".", + "$ref": "meta/core#/$defs/anchorString", + "deprecated": true + }, + "$recursiveRef": { + "$comment": "\"$recursiveRef\" has been replaced by \"$dynamicRef\".", + "$ref": "meta/core#/$defs/uriReferenceString", + "deprecated": true + } + } +} diff --git a/packages/schema/test/shared.test.ts b/packages/schema/test/shared.test.ts new file mode 100644 index 000000000..ba3e4ab30 --- /dev/null +++ b/packages/schema/test/shared.test.ts @@ -0,0 +1,42 @@ +// Values shared across compiled schemas hold nothing alive: a cached pattern goes once its schemas are collected. + +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import { describe, expect, it } from "vitest"; +import { compile } from "../src/index.ts"; +import { sharedRegexCount } from "../src/shared.ts"; + +setFlagsFromString("--expose-gc"); +const gc = runInNewContext("gc") as () => void; + +async function collectGarbage(): Promise { + for (let round = 0; round < 6; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } +} + +describe("shared checks", () => { + it("share the module-level check of a bare type", () => { + const a = compile({ type: "string", description: "a" }); + const b = compile({ type: "string", title: "b" }); + expect(a.check).toBe(b.check); + }); + + it("share a pattern's regular expression and drop it with the last schema that uses it", async () => { + await collectGarbage(); + const before = sharedRegexCount(); + let validators: { check: object }[] | undefined = Array.from({ length: 50 }, (_, i) => + compile({ type: "string", pattern: `^unique-${i}$` }), + ); + const again = compile({ type: "string", pattern: "^unique-0$" }); + expect(sharedRegexCount()).toBe(before + 50); + expect(again.check("unique-0")).toBe(true); + const check = new WeakRef(validators[0].check); + validators = undefined; + await collectGarbage(); + // `again` still uses ^unique-0$; the other 49 patterns are gone. + expect(check.deref()).toBeUndefined(); + expect(sharedRegexCount()).toBe(before + 1); + }); +}); diff --git a/packages/schema/test/suite.ts b/packages/schema/test/suite.ts new file mode 100644 index 000000000..fac5bebbd --- /dev/null +++ b/packages/schema/test/suite.ts @@ -0,0 +1,77 @@ +// Loads the vendored JSON-Schema-Test-Suite: its test files, the remote documents and the official meta-schemas. + +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative } from "node:path"; +import type { JsonSchema } from "../src/index.ts"; + +/** One test file: its path below `tests/` and its groups. */ +export interface SuiteFile { + readonly name: string; + readonly draft: "2020-12" | "7"; + readonly groups: readonly SuiteGroup[]; +} + +/** A schema with values and whether each is valid. */ +export interface SuiteGroup { + readonly description: string; + readonly schema: JsonSchema; + readonly tests: readonly { readonly description: string; readonly data: unknown; readonly valid: boolean }[]; +} + +const fixtures = join(import.meta.dirname, "fixtures"); +const root = join(fixtures, "json-schema-test-suite"); + +/** Every test file of drafts 2020-12 and 7. */ +export function suiteFiles(): SuiteFile[] { + const files: SuiteFile[] = []; + for (const [dir, draft] of [ + ["draft2020-12", "2020-12"], + ["draft7", "7"], + ] as const) { + for (const path of jsonFiles(join(root, "tests", dir))) { + const name = relative(join(root, "tests"), path).replace(/\.json$/, ""); + files.push({ name, draft, groups: JSON.parse(readFileSync(path, "utf8")) }); + } + } + return files; +} + +/** The remote documents and meta-schemas the tests reference, by URI (fresh copies). */ +export function suiteSchemas(): Record { + const schemas: Record = {}; + const remotes = join(root, "remotes"); + for (const path of jsonFiles(remotes)) { + schemas[`http://localhost:1234/${relative(remotes, path).replaceAll("\\", "/")}`] = readJson(path); + } + const meta = join(fixtures, "meta-schemas"); + schemas["http://json-schema.org/draft-07/schema"] = readJson(join(meta, "draft-07-schema.json")); + schemas["https://json-schema.org/draft/2020-12/schema"] = readJson(join(meta, "draft2020-12-schema.json")); + for (const vocabulary of [ + "core", + "applicator", + "unevaluated", + "validation", + "meta-data", + "format-annotation", + "content", + ]) { + schemas[`https://json-schema.org/draft/2020-12/meta/${vocabulary}`] = readJson( + join(meta, `draft2020-12-meta-${vocabulary}.json`), + ); + } + return schemas; +} + +function readJson(path: string): JsonSchema { + return JSON.parse(readFileSync(path, "utf8")); +} + +function jsonFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir).sort()) { + const path = join(dir, entry); + if (statSync(path).isDirectory()) out.push(...jsonFiles(path)); + else if (entry.endsWith(".json")) out.push(path); + } + return out; +} diff --git a/packages/schema/test/typebox-static.test.ts b/packages/schema/test/typebox-static.test.ts new file mode 100644 index 000000000..c62ec03f9 --- /dev/null +++ b/packages/schema/test/typebox-static.test.ts @@ -0,0 +1,72 @@ +// Static for TypeBox-built schemas: the common kinds give TypeBox's own static type, without pi-schema importing +// TypeBox; other kinds give `unknown`. Type-level tests (they pass when this file type-checks), against TypeBox 1.3.27. + +import { Type, type Static as TypeBoxOwn } from "typebox"; +import { describe, expectTypeOf, it } from "vitest"; +import { compile, type Static } from "../src/index.ts"; + +describe("Static of TypeBox schemas", () => { + it("maps the primitive kinds, literals and Unsafe", () => { + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + expectTypeOf>>().toEqualTypeOf(); + const any = Type.Any(); + expectTypeOf>().toBeAny(); + const literal = Type.Literal("add"); + expectTypeOf>().toEqualTypeOf<"add">(); + const unsafe = Type.Unsafe<"a" | "b">({ type: "string", enum: ["a", "b"] }); + expectTypeOf>().toEqualTypeOf<"a" | "b">(); + }); + + it("maps objects with optional and readonly properties as TypeBox does", () => { + const schema = Type.Object({ + path: Type.String(), + offset: Type.Optional(Type.Integer()), + id: Type.Readonly(Type.String()), + tag: Type.Readonly(Type.Optional(Type.Literal("x"))), + }); + type Expected = { path: string; offset?: number; readonly id: string; readonly tag?: "x" }; + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + const empty = Type.Object({}); + expectTypeOf>().toEqualTypeOf>(); + }); + + it("maps arrays, unions and records, nested", () => { + const schema = Type.Object({ + edits: Type.Array(Type.Object({ oldText: Type.String(), newText: Type.String() })), + mode: Type.Union([Type.Literal("text"), Type.Literal("binary"), Type.Null()]), + env: Type.Record(Type.String(), Type.Union([Type.String(), Type.Number()])), + counts: Type.Record(Type.Integer(), Type.Boolean()), + }); + type Expected = { + edits: { oldText: string; newText: string }[]; + mode: "text" | "binary" | null; + env: Record; + counts: Record; + }; + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf>(); + }); + + it("gives unknown for kinds it does not map", () => { + const enumeration = Type.Enum(["a", "b"]); + const tuple = Type.Tuple([Type.String()]); + const intersect = Type.Intersect([Type.Object({ a: Type.String() }), Type.Object({ b: Type.String() })]); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf(); + // Inside a mapped kind, an unmapped kind is unknown only where it stands. + const object = Type.Object({ pair: Type.Tuple([Type.String()]), name: Type.String() }); + expectTypeOf>().toEqualTypeOf<{ pair: unknown; name: string }>(); + }); + + it("types compile() and parse() for TypeBox schemas", () => { + const schema = Type.Object({ n: Type.Number(), s: Type.Optional(Type.String()) }); + const result = compile(schema, { coerce: true }).parse({ n: "1" }); + if (result.ok) expectTypeOf(result.value).toEqualTypeOf<{ n: number; s?: string }>(); + }); +}); diff --git a/packages/schema/test/validator.test.ts b/packages/schema/test/validator.test.ts new file mode 100644 index 000000000..c5ebd6edf --- /dev/null +++ b/packages/schema/test/validator.test.ts @@ -0,0 +1,323 @@ +// compile() and Validator: checks, errors, values, options and schemas from other sources. + +import Type from "typebox"; +import { describe, expect, it } from "vitest"; +import { formats } from "../src/formats/index.ts"; +import { compile } from "../src/index.ts"; + +const read = { + type: "object", + properties: { path: { type: "string" }, offset: { type: "integer", minimum: 1 } }, + required: ["path"], + additionalProperties: false, +}; + +const block = { + anyOf: [ + { + type: "object", + properties: { kind: { const: "text" }, text: { type: "string" } }, + required: ["kind", "text"], + additionalProperties: false, + }, + { + type: "object", + properties: { kind: { const: "image" }, width: { type: "integer", minimum: 0 } }, + required: ["kind", "width"], + additionalProperties: false, + }, + ], +}; + +describe("errors", () => { + it("reports a missing field and an unknown field at the field", () => { + expect(compile(read).errors({ offset: 0, extra: true })).toEqual([ + { path: "/path", keyword: "required", message: "missing" }, + { path: "/offset", keyword: "minimum", message: "expected >= 1" }, + { path: "/extra", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); + + it("is empty for a valid value", () => { + expect(compile(read).errors({ path: "a.ts" })).toEqual([]); + }); + + it("reports the branch a tagged union's tag selects", () => { + const validator = compile({ type: "array", items: block }); + expect( + validator.errors([ + { kind: "text", text: "a" }, + { kind: "image", width: -1 }, + ]), + ).toEqual([{ path: "/1/width", keyword: "minimum", message: "expected >= 0" }]); + expect(validator.errors([{ kind: "video" }])).toEqual([ + { path: "/0/kind", keyword: "anyOf", message: 'expected one of ["text","image"]' }, + ]); + }); + + it("escapes ~ and / in paths as JSON Pointer does", () => { + const validator = compile({ type: "object", properties: { "a/b~c": { type: "string" } } }); + expect(validator.errors({ "a/b~c": 1 })).toEqual([ + { path: "/a~1b~0c", keyword: "type", message: "expected string, got integer" }, + ]); + }); + + it("follows $ref into the referenced schema", () => { + const validator = compile({ + $defs: { point: { type: "object", properties: { x: { type: "number" } }, required: ["x"] } }, + type: "array", + items: { $ref: "#/$defs/point" }, + }); + expect(validator.errors([{ x: 1 }, {}])).toEqual([{ path: "/1/x", keyword: "required", message: "missing" }]); + }); +}); + +describe("values", () => { + it("treats a property set to undefined as absent", () => { + const validator = compile(read); + expect(validator.check({ path: "a.ts", offset: undefined })).toBe(true); + expect(validator.check({ path: undefined })).toBe(false); + }); + + it("checks bytes as Uint8Array with length bounds", () => { + const validator = compile({ type: "object", properties: { data: { type: "bytes", maxLength: 4 } } }); + expect(validator.check({ data: new Uint8Array(4) })).toBe(true); + expect(validator.check({ data: new Uint8Array(5) })).toBe(false); + expect(validator.check({ data: [1, 2] })).toBe(false); + expect(validator.errors({ data: new Uint8Array(5) })).toEqual([ + { path: "/data", keyword: "maxLength", message: "expected at most 4 bytes" }, + ]); + }); + + it("does not treat bytes as an object", () => { + expect(compile({ type: "object" }).check(new Uint8Array(1))).toBe(false); + }); + + it("checks properties named like Object.prototype members as own properties", () => { + const validator = compile({ + type: "object", + properties: { constructor: { type: "string" }, toString: { type: "number" } }, + required: ["constructor"], + additionalProperties: false, + }); + expect(validator.check({})).toBe(false); + expect(validator.check({ constructor: "a" })).toBe(true); + expect(validator.check({ constructor: "a", toString: "b" })).toBe(false); + }); + + it("does not accept every value for a type named like an Object.prototype member", () => { + expect(compile({ type: "constructor" }).check(1)).toBe(false); + }); + + it("checks a detached check function", () => { + const { check } = compile(read); + expect([{ path: "a" }, { path: 1 }].filter(check)).toEqual([{ path: "a" }]); + }); +}); + +describe("options", () => { + it("treats format as an annotation unless formats are given", () => { + const schema = { type: "string", format: "email" }; + expect(compile(schema).check("not an email")).toBe(true); + expect(compile(schema, { formats }).check("not an email")).toBe(false); + expect(compile(schema, { formats }).check("a@example.com")).toBe(true); + }); + + it("resolves $ref to registered documents", () => { + const point = { type: "object", properties: { x: { type: "number" } }, required: ["x"] }; + const validator = compile( + { $ref: "https://example.com/point.json" }, + { schemas: { "https://example.com/point.json": point } }, + ); + expect(validator.check({ x: 1 })).toBe(true); + expect(validator.check({})).toBe(false); + }); + + it("throws for a $ref that names nothing", () => { + expect(() => compile({ $ref: "https://example.com/missing.json" })).toThrow(/Unresolved \$ref/); + expect(() => compile({ $ref: "#/$defs/missing" })).toThrow(/Unresolved \$ref/); + }); + + it("applies draft 7 semantics: $ref overrides its siblings", () => { + const schema = { definitions: { s: { type: "string" } }, $ref: "#/definitions/s", maxLength: 1 }; + expect(compile(schema, { draft: "7" }).check("ab")).toBe(true); + expect(compile(schema).check("ab")).toBe(false); + }); +}); + +describe("schemas", () => { + it("compiles TypeBox schemas as they are", () => { + const validator = compile(Type.Object({ name: Type.String(), tags: Type.Array(Type.String()) })); + expect(validator.check({ name: "a", tags: [] })).toBe(true); + expect(validator.errors({ name: "a", tags: [1] })).toEqual([ + { path: "/tags/0", keyword: "type", message: "expected string, got integer" }, + ]); + }); + + it("compiles deep-frozen schemas and never modifies a schema", () => { + const schema = deepFreeze(structuredClone({ ...block, $defs: { s: { type: "string", pattern: "^a" } } })); + const before = JSON.stringify(schema); + const validator = compile(schema); + expect(validator.check({ kind: "text", text: "x" })).toBe(true); + expect(validator.errors({ kind: "image", width: -1 })).toHaveLength(1); + expect(JSON.stringify(schema)).toBe(before); + }); + + it("compiles a recursive schema", () => { + const tree = { + type: "object", + properties: { value: { type: "number" }, children: { type: "array", items: { $ref: "#" } } }, + required: ["value"], + }; + const validator = compile(tree); + expect(validator.check({ value: 1, children: [{ value: 2, children: [{ value: 3 }] }] })).toBe(true); + expect(validator.errors({ value: 1, children: [{ value: 2, children: [{}] }] })).toEqual([ + { path: "/children/0/children/0/value", keyword: "required", message: "missing" }, + ]); + }); + + it("throws for a malformed schema", () => { + expect(() => compile({ properties: { a: 1 } })).toThrow(/must be an object or a boolean/); + }); +}); + +function deepFreeze(value: T): T { + if (typeof value === "object" && value !== null && !Object.isFrozen(value)) { + Object.freeze(value); + for (const child of Object.values(value)) deepFreeze(child); + } + return value; +} + +describe("unresolved references", () => { + const dangling = { type: "object", properties: { t: { $ref: "#/definitions/Missing" } } }; + const external = { type: "object", properties: { t: { $ref: "https://example.com/schemas/t.json" } } }; + + it("throw when the schema compiles, by default", () => { + expect(() => compile(dangling)).toThrow("Unresolved $ref #/definitions/Missing"); + expect(() => compile(external)).toThrow("Unresolved $ref https://example.com/schemas/t.json"); + }); + + it("reject only values at their place with unresolvedRefs: reject", () => { + for (const schema of [dangling, external]) { + const validator = compile(schema, { unresolvedRefs: "reject" }); + expect(validator.check({})).toBe(true); + expect(validator.check({ t: 1 })).toBe(false); + } + expect(compile(dangling, { unresolvedRefs: "reject" }).errors({ t: 1 })).toEqual([ + { path: "/t", keyword: "$ref", message: "unresolved reference #/definitions/Missing" }, + ]); + }); + + it("leave values at their place uncoerced", () => { + const validator = compile(dangling, { unresolvedRefs: "reject", coerce: true }); + expect(validator.parse({ t: "1" })).toEqual({ + ok: false, + errors: [{ path: "/t", keyword: "$ref", message: "unresolved reference #/definitions/Missing" }], + }); + }); + + it("reject a TypeBox Ref without its definition", () => { + const validator = compile(Type.Object({ a: Type.Ref("A") }), { unresolvedRefs: "reject" }); + expect(validator.errors({ a: { x: "1" } })).toEqual([ + { path: "/a", keyword: "$ref", message: "unresolved reference A" }, + ]); + }); +}); + +describe("string lengths", () => { + it("count an unpaired high surrogate as one code point and a pair as one", () => { + expect(compile({ type: "string", minLength: 2 }).check("\ud800A")).toBe(true); + expect(compile({ type: "string", maxLength: 1 }).check("\ud800A")).toBe(false); + expect(compile({ type: "string", maxLength: 1 }).check("\ud800")).toBe(true); + expect(compile({ type: "string", maxLength: 1 }).check("😀")).toBe(true); + expect(compile({ type: "string", minLength: 2 }).check("😀")).toBe(false); + expect(compile({ type: "string", minLength: 2 }).check("\udc00\ud800")).toBe(true); + }); +}); + +describe("error messages", () => { + it("list at most 20 enum values", () => { + const values = Array.from({ length: 5000 }, (_, i) => `v${i}`); + const validator = compile({ type: "object", properties: { e: { type: "string", enum: values } } }); + expect(validator.check({ e: "v4999" })).toBe(true); + const [error] = validator.errors({ e: "v5000" }); + expect(error.message).toBe( + `expected one of ${JSON.stringify(values.slice(0, 20)).slice(0, -1)}, ... and 4980 more]`, + ); + expect(compile({ enum: ["a", "b"] }).errors("c")).toEqual([ + { path: "", keyword: "enum", message: 'expected one of ["a","b"]' }, + ]); + }); + + it("accept a 10,000-value enum", () => { + const values = Array.from({ length: 10_000 }, (_, i) => `member${i}`); + expect(compile({ enum: values }).check("member9999")).toBe(true); + }); + + it("describe contains bounds in words", () => { + const at = (schema: object) => compile({ type: "array", ...schema }).errors(["a"])[0]?.message; + expect(at({ contains: { type: "number" } })).toBe("expected at least 1 matching item"); + expect(at({ contains: { type: "number" }, minContains: 2 })).toBe("expected at least 2 matching items"); + expect(at({ contains: { type: "string" }, maxContains: 0, minContains: 0 })).toBe( + "expected at most 0 matching items", + ); + expect(at({ contains: { type: "number" }, minContains: 1, maxContains: 3 })).toBe( + "expected between 1 and 3 matching items", + ); + }); + + it("blame a oneOf failure on the branch that declares most of the object's properties", () => { + const filter = { + type: "object", + properties: { where: { $ref: "#/$defs/F" } }, + $defs: { + F: { + oneOf: [ + { + type: "object", + properties: { field: { type: "string" }, eq: { type: "string" } }, + required: ["field", "eq"], + additionalProperties: false, + }, + { + type: "object", + properties: { and: { type: "array", items: { $ref: "#/$defs/F" } } }, + required: ["and"], + additionalProperties: false, + }, + ], + }, + }, + }; + expect(compile(filter).errors({ where: { and: [{ field: "a", eq: 1, x: 2 }] } })).toEqual([ + { path: "/where/and/0/eq", keyword: "type", message: "expected string, got integer" }, + { path: "/where/and/0/x", keyword: "additionalProperties", message: "unknown field" }, + ]); + }); + + it("explain one object reached through the same reference at two paths at each path", () => { + const schema = { + type: "object", + properties: { a: { $ref: "#/$defs/P" }, b: { $ref: "#/$defs/P" } }, + $defs: { P: { type: "object", properties: { n: { type: "integer" } } } }, + }; + const shared = { n: "x" }; + expect(compile(schema).errors({ a: shared, b: shared })).toEqual([ + { path: "/a/n", keyword: "type", message: "expected integer, got string" }, + { path: "/b/n", keyword: "type", message: "expected integer, got string" }, + ]); + }); +}); + +describe("TypeBox's extra formats", () => { + it("check url and json-pointer-uri-fragment", () => { + const url = compile({ type: "string", format: "url" }, { formats }); + expect(url.check("https://example.com/a?b=1")).toBe(true); + expect(url.check("not a url")).toBe(false); + const fragment = compile({ type: "string", format: "json-pointer-uri-fragment" }, { formats }); + expect(fragment.check("#/a/b~1c")).toBe(true); + expect(fragment.check("#/a b")).toBe(false); + expect(fragment.check("/a")).toBe(false); + }); +}); diff --git a/packages/schema/tsconfig.build.json b/packages/schema/tsconfig.build.json new file mode 100644 index 000000000..695dd9ad2 --- /dev/null +++ b/packages/schema/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "./src" + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "**/*.d.ts", "src/**/*.d.ts"] +} diff --git a/packages/schema/vitest.config.ts b/packages/schema/vitest.config.ts new file mode 100644 index 000000000..18733b4d4 --- /dev/null +++ b/packages/schema/vitest.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + reporters: process.env.GITHUB_ACTIONS ? ["dot", "github-actions"] : ["dot"], + }, + resolve: { conditions: ["source"] }, + ssr: { resolve: { conditions: ["source"] } }, +}); diff --git a/packages/telemetry/CHANGELOG.md b/packages/telemetry/CHANGELOG.md index 52617bb21..a6dd650a8 100644 --- a/packages/telemetry/CHANGELOG.md +++ b/packages/telemetry/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + ### Fixed - The package root and `./testing` exports have a `source` condition, like `./render`, so workspace tests that resolve with `source` (Durable's and pi-env's, as the Environments workflow runs them) work without a build. diff --git a/packages/telemetry/package.json b/packages/telemetry/package.json index c571023b6..eeb30b234 100644 --- a/packages/telemetry/package.json +++ b/packages/telemetry/package.json @@ -1,6 +1,6 @@ { "name": "@earendil-works/pi-telemetry", - "version": "1.4.0", + "version": "1.5.0", "description": "Vendor-neutral telemetry contracts and typed schema utilities for pi", "type": "module", "main": "./dist/index.js", @@ -27,7 +27,7 @@ "README.md" ], "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "test": "vitest --run", "prepublishOnly": "npm run clean && npm run build" diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.md index 2f0b69e9a..5c7b22acf 100644 --- a/packages/tui/CHANGELOG.md +++ b/packages/tui/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +## [1.5.0] - 2026-11-02 + ### Added - Added `replaceControlCharacters()` to replace runs of C0 and C1 control characters with a space @@ -10,11 +12,21 @@ ### Fixed +- Fixed `Shift+Enter` queueing a follow-up in Apple Terminal setups that send `ESC` + Return. When local native Shift detection is available (Apple Terminal, native Windows), `ProcessTerminal` now treats `ESC` + Return with Shift held as `Shift+Enter`; without Shift it remains `Alt+Enter` ([#11559](https://github.com/earendil-works/pi/issues/11559)) +- Fixed regular-mode `TUI` writing OSC 133 prompt markers from rendered lines to the terminal. Terminals treat `133;A` as a new shell prompt and drop the OSC 7501 program status, so the working indicator disappeared during runs. Fullscreen still uses the markers for prompt navigation and already stripped them ([#11519](https://github.com/earendil-works/pi/issues/11519)) - Fixed fullscreen double-click selection splitting paths and tokens at `~`, `.`, `\`, and `+`. Word characters now match iTerm2's defaults ([#10980](https://github.com/earendil-works/pi/issues/10980)) - Fixed `fuzzyMatch()` ranking a query scattered over an earlier part of the text above a contiguous match later in it. It now also scores each contiguous occurrence of the query and keeps the best alignment ([#11220](https://github.com/earendil-works/pi/issues/11220), [#11296](https://github.com/earendil-works/pi/pull/11296)) - Fixed `ProcessTerminal.setTitle()` writing control characters from the title, which could end the title sequence and send the rest to the terminal as escape sequences - Fixed display math wider than the content width being wrapped row by row, which split fractions and sums apart. It now falls back to the raw LaTeX source ([#11483](https://github.com/earendil-works/pi/issues/11483)) - Fixed `tui.select.pageUp` and `tui.select.pageDown` being ignored by `SelectList`, `SettingsList`, and editor autocomplete ([#7629](https://github.com/earendil-works/pi/issues/7629), based on [#7680](https://github.com/earendil-works/pi/pull/7680) by [@muyiyr](https://github.com/muyiyr)) +- Fixed `wrapTextWithAnsi()` starting the next line with a space when a styled word ended exactly at the wrap width, and strikethrough continuing into the right padding of wrapped lines ([#10925](https://github.com/earendil-works/pi/issues/10925)) +- Fixed `Markdown` styling, highlighting, and wrapping every block on each render. It now reuses the top-level blocks of its last render, so a streaming message renders only its changed last block, and a resize reuses highlighted code. Reused blocks are held weakly and dropped by `invalidate()`, which a theme change must call because `MarkdownTheme.highlightCode` results are reused until then ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed `Text.setText()` with unchanged text dropping the wrapped lines, so renderers that set the same text on every update wrapped it again ([#10971](https://github.com/earendil-works/pi/issues/10971)) +- Fixed Orca terminals falling back to text for inline images. Orca is now recognized as supporting true color and hyperlinks, and Kitty images when it advertises `ORCA_IMAGE_PROTOCOL=kitty` ([#9329](https://github.com/earendil-works/pi/pull/9329) by [@XRX193](https://github.com/XRX193)) +- Fixed bare Markdown URLs swallowing following CJK punctuation and text; explicit Markdown links and angle autolinks are unchanged ([#10934](https://github.com/earendil-works/pi/pull/10934) by [@haoqixu](https://github.com/haoqixu)) +- Fixed Kitty-protocol F1–F12 shortcuts, including modified function keys, not being recognized, and F13+ and other functional keys inserting invisible private-use characters into the editor ([#11419](https://github.com/earendil-works/pi/issues/11419), [#11345](https://github.com/earendil-works/pi/issues/11345)) +- Fixed Kitty images inside padded or colored boxes disappearing when their first row scrolled above the fullscreen viewport. Partially visible images are now cropped correctly, including beside other text +- Fixed deeply nested Markdown blockquotes producing exponentially many lines, hanging or crashing rendering once their borders exhausted the available width ([#11272](https://github.com/earendil-works/pi/issues/11272)) ## [1.4.0] - 2026-10-21 diff --git a/packages/tui/package.json b/packages/tui/package.json index 31dc4744a..74a49f0d3 100644 --- a/packages/tui/package.json +++ b/packages/tui/package.json @@ -1,11 +1,11 @@ { "name": "@earendil-works/pi-tui", - "version": "1.4.0", + "version": "1.5.0", "description": "Terminal User Interface library with differential rendering for efficient text-based applications", "type": "module", "main": "dist/index.js", "scripts": { - "clean": "shx rm -rf dist", + "clean": "node ../../scripts/package-files.mjs clean dist", "build": "tsc -p tsconfig.build.json", "build:native:darwin": "native/darwin/build.sh", "build:native:linux": "native/linux/build.sh", diff --git a/packages/tui/src/components/markdown.ts b/packages/tui/src/components/markdown.ts index bacfe29f8..a21f75181 100644 --- a/packages/tui/src/components/markdown.ts +++ b/packages/tui/src/components/markdown.ts @@ -1,4 +1,4 @@ -import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens } from "marked"; +import { Marked, type Token, Tokenizer, type TokenizerExtension, type Tokens, type TokensList } from "marked"; import markedCjkFriendly from "marked-cjk-friendly"; import { renderLatex } from "../latex.ts"; import { getCapabilities, hyperlink, isImageLine } from "../terminal-image.ts"; @@ -244,6 +244,7 @@ export interface MarkdownTheme { italic: (text: string) => string; strikethrough: (text: string) => string; underline: (text: string) => string; + /** Highlighted lines of a code block. A Markdown component reuses the result for the same code until invalidate(). */ highlightCode?: (code: string, lang?: string) => string[]; /** Prefix applied to each rendered code block line (default: " ") */ codeBlockIndent?: string; @@ -265,6 +266,16 @@ interface InlineStyleContext { stylePrefix: string; } +/** A top-level block of the last render, keyed by its source, reused by the next render. */ +interface RenderedBlock { + /** Width and type of the following block (which decides the spacing) that the lines were rendered for. */ + width: number; + nextTokenType: string | undefined; + lines: string[]; + /** Highlighted lines of the block's code, keyed by language and source; they survive a resize or new spacing. */ + highlightedCode: Map; +} + export class Markdown implements Component { private text: string; private paddingX: number; // Left/right padding @@ -281,7 +292,14 @@ export class Markdown implements Component { // Parsed tokens depend only on the source, so they survive theme and width invalidation. Held weakly: a token tree is // about ten times the size of its source, and every message of a long transcript keeps a Markdown component. The // tokens survive a burst of re-renders, such as a theme preview, and are collected afterwards. - private cachedTokens?: WeakRef<{ source: string; tokens: Token[] }>; + private cachedTokens?: WeakRef<{ source: string; tokens: TokensList }>; + // The last render's top-level blocks. While a message streams only its last block changes, so a render styles and + // wraps only that block. Each render keeps only the blocks it used, which drops a streamed block's earlier versions. + // Held weakly like the tokens: the blocks repeat the source in their keys and the code in their highlighting, and + // only a burst of renders, such as streaming or a resize, needs them. + private blocks?: WeakRef>; + // Highlighted code of the block being rendered, only set while renderBlock runs. + private blockHighlightedCode?: Map; constructor( text: string, @@ -301,10 +319,17 @@ export class Markdown implements Component { setText(text: string): void { this.text = text; - this.invalidate(); + this.clearRenderCache(); } invalidate(): void { + this.clearRenderCache(); + // The theme may have changed, which changes every styled line and the default style's prefix. + this.blocks = undefined; + this.defaultStylePrefix = undefined; + } + + private clearRenderCache(): void { this.cachedText = undefined; this.cachedWidth = undefined; this.cachedLines = undefined; @@ -322,6 +347,7 @@ export class Markdown implements Component { // Don't render anything if there's no actual text if (!text || text.trim() === "") { + this.blocks = undefined; const result: string[] = []; // Update cache this.cachedText = this.text; @@ -342,65 +368,39 @@ export class Markdown implements Component { this.cachedTokens = new WeakRef({ source: normalizedText, tokens }); } - // Convert tokens to styled terminal output - const renderedLines: string[] = []; - + // A reference link resolves through a definition anywhere in the text, so blocks with the same source can render + // differently; such texts are rare in practice and render without reusing blocks. + const reuseBlocks = Object.keys(tokens.links).length === 0; + const previousBlocks = this.blocks?.deref(); + const blocks = new Map(); + this.blocks = new WeakRef(blocks); + const contentLines: string[] = []; for (let i = 0; i < tokens.length; i++) { const token = tokens[i]; - const nextToken = tokens[i + 1]; - const tokenLines = this.renderToken(token, contentWidth, nextToken?.type); - for (const tokenLine of tokenLines) { - renderedLines.push(tokenLine); + const nextTokenType = tokens[i + 1]?.type; + const reused = reuseBlocks ? (blocks.get(token.raw) ?? previousBlocks?.get(token.raw)) : undefined; + const block = reused ?? { width: -1, nextTokenType, lines: [], highlightedCode: new Map() }; + if (block.width !== width || block.nextTokenType !== nextTokenType) { + this.renderBlock(block, token, nextTokenType, width, contentWidth); } - } - - // Wrap lines (NO padding, NO background yet) - const wrappedLines: string[] = []; - for (const line of renderedLines) { - if (isImageLine(line)) { - wrappedLines.push(line); - } else { - for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { - wrappedLines.push(wrappedLine); - } - } - } - - // Add margins and background to each wrapped line - const leftMargin = " ".repeat(this.paddingX); - const rightMargin = " ".repeat(this.paddingX); - const bgFn = this.defaultTextStyle?.bgColor; - const contentLines: string[] = []; - - for (const line of wrappedLines) { - if (isImageLine(line)) { + if (reuseBlocks) blocks.set(token.raw, block); + for (const line of block.lines) { contentLines.push(line); - continue; - } - - const lineWithMargins = leftMargin + line + rightMargin; - - if (bgFn) { - contentLines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); - } else { - // No background - just pad to width - const visibleLen = visibleWidth(lineWithMargins); - const paddingNeeded = Math.max(0, width - visibleLen); - contentLines.push(lineWithMargins + " ".repeat(paddingNeeded)); } } // Add top/bottom padding (empty lines) + const bgFn = this.defaultTextStyle?.bgColor; const emptyLine = " ".repeat(width); const emptyLines: string[] = []; for (let i = 0; i < this.paddingY; i++) { const line = bgFn ? applyBackgroundToLine(emptyLine, width, bgFn) : emptyLine; emptyLines.push(line); } + flattenLines(emptyLines); // Combine top padding, content, and bottom padding const result = emptyLines.concat(contentLines, emptyLines); - flattenLines(result); // Update cache this.cachedText = this.text; @@ -410,6 +410,42 @@ export class Markdown implements Component { return result.length > 0 ? result : [""]; } + /** Render one top-level block to output lines: wrapped, with margins, and padded or backgrounded to the width. */ + private renderBlock( + block: RenderedBlock, + token: Token, + nextTokenType: string | undefined, + width: number, + contentWidth: number, + ): void { + this.blockHighlightedCode = block.highlightedCode; + const leftMargin = " ".repeat(this.paddingX); + const rightMargin = " ".repeat(this.paddingX); + const bgFn = this.defaultTextStyle?.bgColor; + const lines: string[] = []; + for (const line of this.renderToken(token, contentWidth, nextTokenType)) { + if (isImageLine(line)) { + lines.push(line); + continue; + } + for (const wrappedLine of wrapTextWithAnsi(line, contentWidth)) { + const lineWithMargins = leftMargin + wrappedLine + rightMargin; + if (bgFn) { + lines.push(applyBackgroundToLine(lineWithMargins, width, bgFn)); + } else { + // No background - just pad to width + const paddingNeeded = Math.max(0, width - visibleWidth(lineWithMargins)); + lines.push(lineWithMargins + " ".repeat(paddingNeeded)); + } + } + } + this.blockHighlightedCode = undefined; + flattenLines(lines); + block.lines = lines; + block.width = width; + block.nextTokenType = nextTokenType; + } + /** * Apply default text style to a string. * This is the base styling applied to all text content. @@ -567,8 +603,13 @@ export class Markdown implements Component { const indent = this.theme.codeBlockIndent ?? " "; lines.push(this.theme.codeBlockBorder(`\`\`\`${token.lang || ""}`)); if (this.theme.highlightCode) { - const highlightedLines = this.theme.highlightCode(token.text, token.lang); - for (const hlLine of highlightedLines) { + const key = `${token.lang ?? ""}\0${token.text}`; + let highlighted = this.blockHighlightedCode?.get(key); + if (!highlighted) { + highlighted = this.theme.highlightCode(token.text, token.lang); + this.blockHighlightedCode?.set(key, highlighted); + } + for (const hlLine of highlighted) { lines.push(`${indent}${hlLine}`); } } else { diff --git a/packages/tui/src/components/text.ts b/packages/tui/src/components/text.ts index 024748912..c9aea3d13 100644 --- a/packages/tui/src/components/text.ts +++ b/packages/tui/src/components/text.ts @@ -23,6 +23,8 @@ export class Text implements Component { } setText(text: string): void { + // Renderers often set the same text on every update; keep the wrapped lines then. + if (text === this.text) return; this.text = text; this.cachedText = undefined; this.cachedWidth = undefined; diff --git a/packages/tui/src/terminal-image.ts b/packages/tui/src/terminal-image.ts index 201a40603..64703bf48 100644 --- a/packages/tui/src/terminal-image.ts +++ b/packages/tui/src/terminal-image.ts @@ -96,6 +96,13 @@ function detectCapabilitiesFromEnvironment(tmuxForwardsHyperlink: () => boolean) return { images: "kitty", trueColor: true, hyperlinks: true }; } + // Orca supports Kitty graphics but only advertises it through ORCA_IMAGE_PROTOCOL. Require + // both signals so Orca releases predating that advertisement keep the text fallback instead + // of reserving image rows that nothing renders. + if (termProgram === "orca" && process.env.ORCA_IMAGE_PROTOCOL === "kitty") { + return { images: "kitty", trueColor: true, hyperlinks: true }; + } + if (process.env.KITTY_WINDOW_ID || termProgram === "kitty") { return { images: "kitty", trueColor: true, hyperlinks: true }; } diff --git a/packages/tui/src/terminal.ts b/packages/tui/src/terminal.ts index 8e56022e9..55dab58d5 100644 --- a/packages/tui/src/terminal.ts +++ b/packages/tui/src/terminal.ts @@ -60,12 +60,19 @@ export function refreshTerminalDimensions(): void { } } +// Some Apple Terminal setups send ESC+CR (Alt+Enter) for Shift+Enter (#11559). +function isNativeShiftEnterCandidate(data: string): boolean { + return data === "\r" || data === "\x1b\r"; +} + export function normalizeNativeShiftEnterInput( data: string, shouldDetectNativeShiftEnter: boolean, isShiftPressed: boolean, ): string { - if (shouldDetectNativeShiftEnter && data === "\r" && isShiftPressed) return NATIVE_SHIFT_ENTER_SEQUENCE; + if (shouldDetectNativeShiftEnter && isNativeShiftEnterCandidate(data) && isShiftPressed) { + return NATIVE_SHIFT_ENTER_SEQUENCE; + } return data; } @@ -397,7 +404,7 @@ export class ProcessTerminal implements Terminal { private forwardInputSequence(sequence: string): void { if (!this.inputHandler) return; const shouldDetectNativeShiftEnter = - sequence === "\r" && (isAppleTerminalSession() || process.platform === "win32"); + isNativeShiftEnterCandidate(sequence) && (isAppleTerminalSession() || process.platform === "win32"); const input = normalizeNativeShiftEnterInput( sequence, shouldDetectNativeShiftEnter, diff --git a/packages/tui/src/tui-main-screen.ts b/packages/tui/src/tui-main-screen.ts index c497acdbf..71229fcb5 100644 --- a/packages/tui/src/tui-main-screen.ts +++ b/packages/tui/src/tui-main-screen.ts @@ -5,6 +5,9 @@ import { deleteKittyImage, isImageLine } from "./terminal-image.ts"; import { type TUI, TuiBase, type TuiStopOptions } from "./tui.ts"; import { visibleWidth } from "./utils.ts"; +/** OSC 133 prompt markers at the start of a rendered line. Fullscreen uses them for prompt navigation. */ +const OSC133_ZONE_PREFIX = /^(?:\x1b\]133;[ABC](?:\x07|\x1b\\))+/; + const KITTY_SEQUENCE_PREFIX = "\x1b_G"; const MAX_RENDER_WRITE_CHARS = 1024 * 1024; @@ -262,7 +265,8 @@ export class TuiMainScreen extends TuiBase implements TUI { // Render all components to get new lines // Resolve fake cursors before compositing so overlays and line slicing see plain SGR codes - let newLines = this.resolveFakeCursors(this.render(width)); + // Drop OSC 133 markers: the terminal reads 133;A as a new shell prompt, which ends program status (OSC 7501) + let newLines = this.resolveFakeCursors(this.render(width).map((line) => line.replace(OSC133_ZONE_PREFIX, ""))); // Composite overlays into the rendered lines (before differential compare) if (this.hasOverlayEntries) { diff --git a/packages/tui/src/utils.ts b/packages/tui/src/utils.ts index e8a49e24c..2a84d830d 100644 --- a/packages/tui/src/utils.ts +++ b/packages/tui/src/utils.ts @@ -326,6 +326,49 @@ export function stripTerminalSequences(str: string): string { return result; } +function extractAnsiCodes(str: string): string { + let result = ""; + let i = str.indexOf("\x1b"); + while (i !== -1) { + const length = ansiCodeLength(str, i); + if (length > 0) { + result += str.substring(i, i + length); + i += length; + } else { + i++; + } + i = str.indexOf("\x1b", i); + } + return result; +} + +// Same character set as String.prototype.trim() +const WHITESPACE_REGEX = /\s/; + +/** + * Whether a token's visible text is only whitespace. Codes closing a styled word attach to the following space, + * e.g. "\x1b[22m ", so they are skipped. Stops at the first visible non-whitespace text without allocating a + * stripped copy, because styled words reach this check on every wrap. + */ +function isWhitespaceToken(token: string): boolean { + let i = 0; + while (i < token.length) { + const code = token.charCodeAt(i); + if (code === 0x1b) { + const length = ansiCodeLength(token, i); + if (length === 0) return false; // A lone ESC is visible text + i += length; + } else if (code === 0x20 || (code >= 0x09 && code <= 0x0d)) { + i++; + } else if (code < 0x80 || !WHITESPACE_REGEX.test(token[i]!)) { + return false; + } else { + i++; + } + } + return true; +} + const CONTROL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f]+/g; /** Replace each run of C0 and C1 control characters (ESC, BEL, newlines, ...) with one space. */ @@ -759,7 +802,7 @@ class AnsiCodeTracker { /** * Get reset codes for attributes that need to be turned off at line end. - * Underline must be closed to prevent bleeding into padding. + * Underline and strikethrough must be closed to prevent bleeding into padding. * Active OSC 8 hyperlinks must be closed and re-opened on the next line. * Returns empty string if no attributes need closing. */ @@ -768,6 +811,9 @@ class AnsiCodeTracker { if (this.underline) { result += "\x1b[24m"; // Underline off only } + if (this.strikethrough) { + result += "\x1b[29m"; // Strikethrough off only, re-opened at line start via getActiveCodes() + } if (this.activeHyperlink) { result += formatOsc8Close(this.activeHyperlink.terminator); // Re-opened at line start via getActiveCodes() } @@ -948,11 +994,13 @@ function wrapSingleLine(line: string, width: number): string[] { for (const token of tokens) { const tokenVisibleLength = visibleWidth(token); - const isWhitespace = token.trim() === ""; + const isWhitespace = isWhitespaceToken(token); // Token itself is too long - break it character by character if (tokenVisibleLength > width && !isWhitespace) { if (currentLine) { + // Trim before adding the reset, which would otherwise block the final trimEnd() + currentLine = currentLine.trimEnd(); // Add specific reset for underline only (preserves background) const lineEndReset = tracker.getLineEndReset(); if (lineEndReset) { @@ -979,6 +1027,12 @@ function wrapSingleLine(line: string, width: number): string[] { if (totalNeeded > width && currentVisibleLength > 0) { // Trim trailing whitespace, then add underline reset (not full reset, to preserve background) let lineToWrap = currentLine.trimEnd(); + if (isWhitespace) { + // The dropped space may carry codes (e.g. closing a styled word). Keep them on the wrapped line so + // the style ends there and is not re-opened on the next line. + lineToWrap += extractAnsiCodes(token); + updateTrackerFromText(token, tracker); + } const lineEndReset = tracker.getLineEndReset(); if (lineEndReset) { lineToWrap += lineEndReset; @@ -988,10 +1042,14 @@ function wrapSingleLine(line: string, width: number): string[] { // Don't start new line with whitespace currentLine = tracker.getActiveCodes(); currentVisibleLength = 0; - } else { - currentLine = tracker.getActiveCodes() + token; - currentVisibleLength = tokenVisibleLength; + continue; } + currentLine = tracker.getActiveCodes() + token; + currentVisibleLength = tokenVisibleLength; + } else if (isWhitespace && totalNeeded > width && token.includes("\x1b")) { + // Whitespace wider than an empty line. Plain spaces are removed by the final trimEnd(), but codes after + // them (e.g. a trailing reset) would block that and leave the line wider than width. Keep only the codes. + currentLine += extractAnsiCodes(token); } else { // Add to current line currentLine += token; diff --git a/packages/tui/test/markdown.test.ts b/packages/tui/test/markdown.test.ts index 0cdcf172b..e0f5b4dd5 100644 --- a/packages/tui/test/markdown.test.ts +++ b/packages/tui/test/markdown.test.ts @@ -1,5 +1,7 @@ import assert from "node:assert"; import { afterEach, describe, it } from "node:test"; +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; import type { Terminal as XtermTerminalType } from "@xterm/headless"; import { Chalk } from "chalk"; import { Markdown, type MarkdownTheme } from "../src/components/markdown.ts"; @@ -65,6 +67,63 @@ describe("Markdown component", () => { }); }); + describe("Block reuse", () => { + afterEach(() => resetCapabilitiesCache()); + + function countingTheme(calls: string[]): MarkdownTheme { + return { + ...defaultMarkdownTheme, + highlightCode: (code, lang) => { + calls.push(`${lang}:${code}`); + return code.split("\n").map((line) => chalk.green(line)); + }, + }; + } + + it("highlights a code block once across streamed text and resizes, and again after invalidate", () => { + const calls: string[] = []; + const markdown = new Markdown("```ts\nx\n```", 0, 0, countingTheme(calls)); + markdown.render(80); + markdown.setText("```ts\nx\n```\n\nafter"); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x"]); + markdown.invalidate(); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x", "ts:x"]); + }); + + it("holds rendered blocks weakly, so a full garbage collection releases them", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const calls: string[] = []; + const markdown = new Markdown("```ts\nx\n```", 0, 0, countingTheme(calls)); + markdown.render(80); + // A WeakRef keeps its target alive until the current job ends. + await new Promise((resolve) => setImmediate(resolve)); + gc(); + markdown.render(40); + assert.deepStrictEqual(calls, ["ts:x", "ts:x"]); + }); + + it("renders a block again when the block after it changes its spacing", () => { + const markdown = new Markdown("# Title", 0, 0, defaultMarkdownTheme); + markdown.render(80); + markdown.setText("# Title\nfoo"); + assert.deepStrictEqual( + markdown.render(80).map((line) => stripAnsi(line).trim()), + ["Title", "", "foo"], + ); + }); + + it("follows changed reference link definitions", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: false }); + const markdown = new Markdown("[a][x]\n\n[x]: https://one.example", 0, 0, defaultMarkdownTheme); + markdown.render(80); + markdown.setText("[a][x]\n\n[x]: https://two.example"); + assert.ok(stripAnsi(markdown.render(80).join("\n")).includes("https://two.example")); + }); + }); + describe("Lists", () => { it("should render simple nested list", () => { const markdown = new Markdown( @@ -1689,6 +1748,22 @@ bar`, }); }); + // #10925 + describe("styled words at the wrap boundary", () => { + it("should not start the next line with a space or carry styles into padding", () => { + const markdown = new Markdown("aaaa **bold** next ~~strk~~ more words", 1, 0, defaultMarkdownTheme, { + bgColor: (text) => `\x1b[44m${text}\x1b[49m`, + }); + + assert.deepStrictEqual(markdown.render(11), [ + "\x1b[44m aaaa \x1b[1mbold\x1b[22m \x1b[49m", + "\x1b[44m next \x1b[9mstrk\x1b[29m \x1b[49m", + "\x1b[44m more \x1b[49m", + "\x1b[44m words \x1b[49m", + ]); + }); + }); + // #10154 describe("CJK emphasis", () => { const tagTheme: MarkdownTheme = { diff --git a/packages/tui/test/terminal-image.test.ts b/packages/tui/test/terminal-image.test.ts index 6fd05f43d..0ae5bd8b4 100644 --- a/packages/tui/test/terminal-image.test.ts +++ b/packages/tui/test/terminal-image.test.ts @@ -34,6 +34,7 @@ import { visibleWidth } from "../src/utils.ts"; const ENV_KEYS = [ "TERM", "TERM_PROGRAM", + "ORCA_IMAGE_PROTOCOL", "TERMINAL_EMULATOR", "COLORTERM", "TMUX", @@ -343,6 +344,22 @@ describe("detectCapabilities", () => { }); }); + it("keeps images disabled for Orca without an image capability signal", () => { + withEnv({ TERM_PROGRAM: "Orca" }, () => { + const caps = detectCapabilities(); + assert.strictEqual(caps.images, null); + }); + }); + + it("enables images and hyperlinks for Orca advertising Kitty support", () => { + withEnv({ TERM_PROGRAM: "Orca", ORCA_IMAGE_PROTOCOL: "kitty" }, () => { + const caps = detectCapabilities(); + assert.strictEqual(caps.images, "kitty"); + assert.strictEqual(caps.trueColor, true); + assert.strictEqual(caps.hyperlinks, true); + }); + }); + it("enables hyperlinks for Ghostty", () => { withEnv({ TERM_PROGRAM: "ghostty" }, () => { const caps = detectCapabilities(); diff --git a/packages/tui/test/terminal.test.ts b/packages/tui/test/terminal.test.ts index de3bc40eb..8f8ce0e4f 100644 --- a/packages/tui/test/terminal.test.ts +++ b/packages/tui/test/terminal.test.ts @@ -36,6 +36,13 @@ describe("normalizeNativeShiftEnterInput", () => { assert.equal(normalizeNativeShiftEnterInput("\r", true, true), "\x1b[13;2u"); }); + it("rewrites ESC+CR to CSI-u Shift+Enter when Shift is pressed", () => { + // #11559 + assert.equal(normalizeNativeShiftEnterInput("\x1b\r", true, true), "\x1b[13;2u"); + assert.equal(normalizeNativeShiftEnterInput("\x1b\r", true, false), "\x1b\r"); + assert.equal(normalizeNativeShiftEnterInput("\x1b\r", false, true), "\x1b\r"); + }); + it("leaves Return unchanged when native Shift detection is disabled", () => { assert.equal(normalizeNativeShiftEnterInput("\r", false, true), "\r"); }); diff --git a/packages/tui/test/text.test.ts b/packages/tui/test/text.test.ts new file mode 100644 index 000000000..9c2e2bd61 --- /dev/null +++ b/packages/tui/test/text.test.ts @@ -0,0 +1,12 @@ +import assert from "node:assert"; +import { describe, it } from "node:test"; +import { Text } from "../src/components/text.ts"; + +describe("Text", () => { + it("keeps its wrapped lines when the same text is set again", () => { + const text = new Text("hello world", 0, 0); + const lines = text.render(5); + text.setText("hello world"); + assert.strictEqual(text.render(5), lines); + }); +}); diff --git a/packages/tui/test/tui-render.test.ts b/packages/tui/test/tui-render.test.ts index 3749eac76..c01142a4a 100644 --- a/packages/tui/test/tui-render.test.ts +++ b/packages/tui/test/tui-render.test.ts @@ -209,6 +209,24 @@ describe("TUI bounded render output", () => { }); }); +describe("TUI OSC 133 markers", () => { + it("does not write OSC 133 prompt markers to the terminal", () => { + // Terminals read 133;A as a new shell prompt and drop the OSC 7501 program status. + const terminal = new BoundedWriteTerminal(); + const tui = new TuiMainScreen(terminal); + const component = new TestComponent(); + tui.addChild(component); + component.lines = ["\x1b]133;A\x07prompt", "\x1b]133;B\x07\x1b]133;C\x07output"]; + tui.renderNow(); + component.lines = [...component.lines, "\x1b]133;A\x1b\\reply", "\x1b]133;B\x1b\\\x1b]133;C\x1b\\done"]; + tui.renderNow(); + + const output = terminal.writes.join(""); + assert.ok(!output.includes("\x1b]133;"), "regular mode must not emit OSC 133 markers"); + for (const text of ["prompt", "output", "reply", "done"]) assert.ok(output.includes(text)); + }); +}); + /** Set each environment variable to `value`, returning a function that restores the previous state. */ function overrideEnv(names: readonly string[], value: string): () => void { const previousValues = names.map((name) => [name, process.env[name]] as const); diff --git a/packages/tui/test/wrap-ansi.test.ts b/packages/tui/test/wrap-ansi.test.ts index 618be2347..60cfc7248 100644 --- a/packages/tui/test/wrap-ansi.test.ts +++ b/packages/tui/test/wrap-ansi.test.ts @@ -51,6 +51,57 @@ describe("wrapTextWithAnsi", () => { }); }); + // #10925 + describe("styled words at the wrap boundary", () => { + it("keeps the closing code on the wrapped line and drops the following space", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaaa \x1b[1mbold\x1b[22m next word", 9), [ + "aaaa \x1b[1mbold\x1b[22m", + "next word", + ]); + }); + + it("does not re-open a style closed by the dropped space", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaaa \x1b[31mred\x1b[39m next", 8), [ + "aaaa \x1b[31mred\x1b[39m", + "next", + ]); + }); + + it("does not exceed width for styled whitespace wider than the line", () => { + for (const text of [ + `\x1b[41m${" ".repeat(20)}\x1b[0m`, + `ab\x1b[41m${" ".repeat(20)}\x1b[49mcd`, + `\x1b[33m${" ".repeat(20)}x\x1b[39m`, + ]) { + for (const line of wrapTextWithAnsi(text, 10)) { + assert.ok(visibleWidth(line) <= 10, JSON.stringify(line)); + } + } + }); + + it("trims trailing spaces before the line-end reset when a long word follows", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaa\x1b[9m abcdefghij\x1b[29m", 9), [ + "aaa\x1b[9m\x1b[29m", + "\x1b[9mabcdefghi\x1b[29m", + "\x1b[9mj\x1b[29m", + ]); + }); + + it("resets strikethrough at line end and restores it on the next line", () => { + assert.deepStrictEqual(wrapTextWithAnsi("aaaa \x1b[9mstrike me\x1b[29m next", 12), [ + "aaaa \x1b[9mstrike\x1b[29m", + "\x1b[9mme\x1b[29m next", + ]); + }); + + it("preserves background when resetting strikethrough at line end", () => { + assert.deepStrictEqual(wrapTextWithAnsi("\x1b[41;9mstrike me\x1b[0m", 6), [ + "\x1b[41;9mstrike\x1b[29m", + "\x1b[9;41mme\x1b[0m", + ]); + }); + }); + describe("background color preservation", () => { it("should preserve background color across wrapped lines without full reset", () => { const bgBlue = "\x1b[44m"; diff --git a/scripts/browser-smoke-entry.ts b/scripts/browser-smoke-entry.ts index 4c6d07c38..9cf69450d 100644 --- a/scripts/browser-smoke-entry.ts +++ b/scripts/browser-smoke-entry.ts @@ -1,12 +1,17 @@ import { createHost, memoryLinkPair, replicatedState } from "@earendil-works/chord"; -import { createAssistantMessageEventStream, Type } from "@earendil-works/pi-ai"; +import { createAssistantMessageEventStream, validateToolArguments } from "@earendil-works/pi-ai"; import { complete, getModel, getProviders, streamSimple } from "@earendil-works/pi-ai/compat"; import { Agent, streamProxy } from "@earendil-works/pi-agent-core"; +import { obj, str } from "@earendil-works/pi-schema"; // Keep this entry browser-safe. It is bundled by scripts/check-browser-smoke.mjs // to catch accidental Node-only runtime imports in browser-facing package exports. const model = getModel("google", "gemini-2.5-flash"); -const schema = Type.Object({ prompt: Type.String() }); +const schema = obj({ prompt: str() }); +const args = validateToolArguments( + { name: "prompt", description: "", parameters: schema }, + { type: "toolCall", id: "1", name: "prompt", arguments: { prompt: 1 } }, +); const stream = createAssistantMessageEventStream(); const agent = new Agent({ initialState: { model }, streamFn: streamSimple }); @@ -17,6 +22,7 @@ console.log( getProviders().length, typeof complete, schema.type, + args.prompt, typeof stream.push, agent.hasQueuedMessages(), typeof streamProxy, diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index b3b033a73..4aea32757 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -26,6 +26,7 @@ const WORKSPACE = { "@earendil-works/pi-codemode": "packages/codemode/src", "@earendil-works/pi-telemetry": "packages/telemetry/src", "@earendil-works/pi-mcp": "packages/mcp/src", + "@earendil-works/pi-schema": "packages/schema/src", "@earendil-works/pi-tui": "packages/tui/src", }; @@ -34,15 +35,27 @@ const WORKSPACE = { * graph it is allowed to reach. */ const BUDGETS = { + "packages/schema": { + // The format checks (vendored from TypeBox, about 40 KB) load only through ./formats. + ".": { maxFiles: 10, forbid: ["schema/src/formats/"] }, + }, "packages/ai": { "./models": { // Request telemetry reaches pi-ai's telemetry.ts and pi-telemetry's root (index, noop, metrics, memory). // Request identity reaches utils/identity.ts and its version constant. maxFiles: 20, - forbid: ["providers/", "models.generated.ts", "index.ts", "utils/validation.ts", "utils/typebox-helpers.ts"], + forbid: ["providers/", "models.generated.ts", "index.ts", "utils/validation.ts", "utils/string-enum.ts"], allow: ["packages/telemetry/src/index.ts"], }, - "./utils/*": { maxFiles: 3, forbid: ["providers/", "api/", "index.ts"] }, + "./utils/*": { + maxFiles: 3, + forbid: ["providers/", "api/", "index.ts"], + overrides: { + // Tool argument validation runs on pi-schema: its validator, coercion and format checks (the checks TypeBox's + // validator ran by default). TypeBox, an external package, was not counted before. + "./utils/validation": { maxFiles: 42, forbid: ["providers/", "api/", "packages/ai/src/index.ts"] }, + }, + }, }, "packages/codemode": { // Node and Bun users never load the Cloudflare entry, which imports the `cloudflare:workers` built-in. @@ -60,24 +73,30 @@ const BUDGETS = { }, "packages/durable": { ".": { - // Tool argument validation reaches TypeBox; provider-session creation reaches pi-ai's lean UUID utility; the + // Tool argument validation reaches pi-schema (validator, coercion and format checks; 39 files that replace + // TypeBox, which this walk did not count); provider-session creation reaches pi-ai's lean UUID utility; the // memory and SQLite storages share scan order and cursor handling. Values from Chord's root (copyJson, // replicatedState) reach its host runtime through the root barrel. Chord's context reaches pi-telemetry's root // (index, noop, metrics, memory) for its NOOP telemetry default; harness/telemetry.ts is Durable's schema. // The Harness's request identity reaches pi-ai's utils/identity.ts and its version constant. Partial commits // parse streamed tool-call arguments with pi-ai's utils/streaming-json.ts (and its json-parse.ts) to append what // argument strings grew by (harness/partial-changes.ts). Chord's watches fold pending ops with Delta's - // pending queue (delta/pending.ts). - maxFiles: 77, + // pending queue (delta/pending.ts). `registryReader` (harness/registry-reader.ts) reads a Chord v2 registry + // value through type-only imports. The Session caches hot Storage reads (session/cached-storage.ts) and keeps + // each conversation's newest entry and head marker (session/conversation-tails.ts). Group commit sends batches + // through `commit()`, or one `commit()` each (storage/commit-batch.ts), from a writer off the Session line + // (session/writer.ts) that gates reads by the keys of unpublished writes (session/keys.ts). + maxFiles: 123, forbid: [ "packages/ai/src/index.ts", - "packages/ai/src/utils/typebox-helpers.ts", + "packages/ai/src/utils/string-enum.ts", // Photon (WebAssembly) loads only through `./images`. "packages/durable/src/images/", ], }, - // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself. - "./tools": { maxFiles: 17, forbid: ["packages/durable/src/images/"] }, + // The file tools reach their environment helpers and the Harness definitions they need, not the Harness itself, + // and pi-schema's root for their schema builders (10 files; its format checks stay out). + "./tools": { maxFiles: 27, forbid: ["packages/durable/src/images/", "packages/schema/src/formats/"] }, }, }; @@ -147,6 +166,8 @@ for (const [pkgDir, budgets] of Object.entries(BUDGETS)) { } const target = typeof declared === "string" ? declared : declared.import; for (const [name, distPath] of expand(pkgDir, entry, target)) { + // A wildcard entry's budget applies to every file it exports, unless `overrides` names that file. + const limits = budget.overrides?.[name] ?? budget; const source = sourceFor(pkgDir, distPath); if (!source) { console.error(`${pkgDir} export "${name}" points at ${distPath}, which has no source file`); @@ -154,15 +175,15 @@ for (const [pkgDir, budgets] of Object.entries(BUDGETS)) { continue; } const graph = [...walk(source)].map((file) => relative(ROOT, file).split(sep).join("/")); - if (graph.length > budget.maxFiles) { + if (graph.length > limits.maxFiles) { console.error( - `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${budget.maxFiles}\n` + + `${pkgDir} export "${name}" reaches ${graph.length} files, budget ${limits.maxFiles}\n` + graph.map((file) => ` ${file}`).join("\n"), ); failures += 1; } - for (const pattern of budget.forbid ?? []) { - const hit = graph.filter((file) => file.includes(pattern) && !(budget.allow ?? []).some((allowed) => file.endsWith(allowed))); + for (const pattern of limits.forbid ?? []) { + const hit = graph.filter((file) => file.includes(pattern) && !(limits.allow ?? []).some((allowed) => file.endsWith(allowed))); if (hit.length > 0) { console.error(`${pkgDir} export "${name}" must not reach ${pattern}:\n${hit.map((f) => ` ${f}`).join("\n")}`); failures += 1; diff --git a/scripts/model-catalog-protocol.test.ts b/scripts/model-catalog-protocol.test.ts index cdfcc0d50..0a7e3af77 100644 --- a/scripts/model-catalog-protocol.test.ts +++ b/scripts/model-catalog-protocol.test.ts @@ -55,25 +55,23 @@ test("selects the newest catalog a Pi version supports", () => { assert.equal(selectModelCatalog(index, undefined)?.revision, mixedApiRevision); }); -test("redirects Pi user agents to an explicit catalog version", () => { +test("serves Pi user agents the catalog for their version", () => { // Regression test for #9099: released clients identify themselves only by User-Agent. assert.deepEqual( parseModelCatalogRequest( "https://pi.dev/api/models/providers/openrouter?types=chat%2Cimage", "pi/0.84.4 (linux; node/v22.0.0; x64)", ), - { - kind: "redirect", - location: "https://pi.dev/api/models/providers/openrouter?types=chat%2Cimage&pi-version=0.84.4", - }, + { kind: "catalog", piVersion: "0.84.4", representation: "typed" }, ); assert.deepEqual(parseModelCatalogRequest("https://pi.dev/api/models", "pi/0.84.4"), { - kind: "redirect", - location: "https://pi.dev/api/models?pi-version=0.84.4", + kind: "catalog", + piVersion: "0.84.4", + representation: "legacy", }); }); -test("serves explicit and unversioned catalog requests without redirecting", () => { +test("serves explicit and unversioned catalog requests", () => { assert.deepEqual( parseModelCatalogRequest("https://pi.dev/api/models?pi-version=0.83.0", "pi/0.85.1 (linux; node/v22.0.0; x64)"), { kind: "catalog", piVersion: "0.83.0", representation: "legacy" }, diff --git a/scripts/model-catalog-protocol.ts b/scripts/model-catalog-protocol.ts index d151d36a5..173c62b19 100644 --- a/scripts/model-catalog-protocol.ts +++ b/scripts/model-catalog-protocol.ts @@ -53,7 +53,6 @@ export type ModelCatalogIndex = { export type ModelCatalogRequest = | { kind: "catalog"; piVersion: string | undefined; representation: ModelCatalogRepresentation } - | { kind: "redirect"; location: string } | { kind: "invalid"; error: string }; export function getModelCatalogArtifactKey(revision: string, artifact: ModelCatalogArtifact): string { @@ -97,10 +96,11 @@ export function parseModelCatalogRepresentation( /** * Decide how to answer a catalog request from its URL and User-Agent. * - * Released clients do not send `pi-version`. Responses are cached by URL, so - * instead of varying the response on the User-Agent, clients that identify - * as Pi are redirected to the equivalent URL with an explicit `pi-version`. - * Requests without a Pi User-Agent receive the default catalog. + * Released clients do not send `pi-version`, so clients that identify as Pi + * are served the catalog for the version in their User-Agent. The response + * therefore varies on the User-Agent, and the server must say so with `Vary`. + * An explicit `pi-version` takes precedence. Requests without a Pi User-Agent + * receive the default catalog. */ export function parseModelCatalogRequest(url: string | URL, userAgent: string | null | undefined): ModelCatalogRequest { const requestUrl = new URL(url); @@ -113,8 +113,7 @@ export function parseModelCatalogRequest(url: string | URL, userAgent: string | if (piVersion === null) { const userAgentVersion = PI_USER_AGENT_RE.exec(userAgent ?? "")?.[1]; if (userAgentVersion !== undefined && isValidModelCatalogPiVersion(userAgentVersion)) { - requestUrl.searchParams.set("pi-version", userAgentVersion); - return { kind: "redirect", location: requestUrl.toString() }; + return { kind: "catalog", piVersion: userAgentVersion, representation }; } return { kind: "catalog", piVersion: undefined, representation }; } diff --git a/scripts/npm-audit.mjs b/scripts/npm-audit.mjs index f636d2347..f2d391c90 100644 --- a/scripts/npm-audit.mjs +++ b/scripts/npm-audit.mjs @@ -10,15 +10,7 @@ const auditLevel = "moderate"; const severities = ["info", "low", "moderate", "high", "critical"]; /** @type {Record} */ -const acceptedAdvisories = { - "GHSA-86w9-cpqp-85rv": { - package: "node-forge", - reason: - "No fixed node-forge release exists. It is only reachable through @earendil-works/gondolin in the private, " + - "unpublished gondolin example extension. Gondolin only verifies leaf certificates against its own locally " + - "generated CA (public exponent 65537), so the low-exponent signature forgery does not apply.", - }, -}; +const acceptedAdvisories = {}; const result = spawnSync("npm", ["audit", "--omit=dev", "--json"], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024 }); if (result.error) { diff --git a/scripts/package-files.mjs b/scripts/package-files.mjs new file mode 100644 index 000000000..6e64aa95f --- /dev/null +++ b/scripts/package-files.mjs @@ -0,0 +1,30 @@ +// Cross-platform file operations for package build scripts, using only Node built-ins. +import { chmodSync, cpSync, globSync, mkdirSync, rmSync, statSync } from "node:fs"; +import { basename, join } from "node:path"; + +const [command, ...paths] = process.argv.slice(2); +const usage = "Usage: package-files.mjs clean | copy | executable "; + +if (paths.length === 0) throw new Error(usage); + +switch (command) { + case "clean": + for (const path of paths) rmSync(path, { recursive: true, force: true }); + break; + case "copy": { + const destination = paths.pop(); + if (!destination || paths.length === 0) throw new Error(usage); + mkdirSync(destination, { recursive: true }); + for (const pattern of paths) { + const sources = globSync(pattern); + if (sources.length === 0) throw new Error(`No files match ${pattern}`); + for (const source of sources) cpSync(source, join(destination, basename(source)), { recursive: true }); + } + break; + } + case "executable": + for (const path of paths) chmodSync(path, statSync(path).mode | 0o111); + break; + default: + throw new Error(usage); +} diff --git a/scripts/package-files.test.mjs b/scripts/package-files.test.mjs new file mode 100644 index 000000000..9b455a792 --- /dev/null +++ b/scripts/package-files.test.mjs @@ -0,0 +1,82 @@ +// Exercises the file operations used by package build scripts without touching build output. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("./package-files.mjs", import.meta.url)); + +function workspace(t) { + const directory = mkdtempSync(join(tmpdir(), "pi-package-files-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + return directory; +} + +function run(directory, ...args) { + return spawnSync(process.execPath, [script, ...args], { cwd: directory, encoding: "utf8" }); +} + +test("clean removes directories and files and ignores missing paths", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "dist", "nested"), { recursive: true }); + writeFileSync(join(directory, "dist", "nested", "file.js"), "output"); + writeFileSync(join(directory, "output.txt"), "output"); + assert.equal(run(directory, "clean", "dist", "output.txt", "missing").status, 0); + assert.equal(existsSync(join(directory, "dist")), false); + assert.equal(existsSync(join(directory, "output.txt")), false); +}); + +test("copy expands patterns, creates directories, and overwrites existing assets", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "source assets")); + writeFileSync(join(directory, "source assets", "dark.json"), "dark"); + writeFileSync(join(directory, "source assets", "light.json"), "light"); + writeFileSync(join(directory, "source assets", "ignored.txt"), "ignored"); + const args = ["copy", "source assets/*.json", "output/assets"]; + assert.equal(run(directory, ...args).status, 0); + assert.equal(readFileSync(join(directory, "output/assets/dark.json"), "utf8"), "dark"); + assert.equal(readFileSync(join(directory, "output/assets/light.json"), "utf8"), "light"); + assert.equal(existsSync(join(directory, "output/assets/ignored.txt")), false); + writeFileSync(join(directory, "source assets", "dark.json"), "updated"); + assert.equal(run(directory, ...args).status, 0); + assert.equal(readFileSync(join(directory, "output/assets/dark.json"), "utf8"), "updated"); +}); + +test("copy accepts multiple files and preserves directory names and nested contents", (t) => { + const directory = workspace(t); + mkdirSync(join(directory, "docs", "nested"), { recursive: true }); + writeFileSync(join(directory, "docs", "nested", "guide.md"), "guide"); + writeFileSync(join(directory, "README.md"), "readme"); + assert.equal(run(directory, "copy", "docs", "README.md", "dist").status, 0); + assert.equal(readFileSync(join(directory, "dist", "docs", "nested", "guide.md"), "utf8"), "guide"); + assert.equal(readFileSync(join(directory, "dist", "README.md"), "utf8"), "readme"); +}); + +test("copy fails when an asset pattern matches nothing", (t) => { + const result = run(workspace(t), "copy", "missing/*.json", "dist"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /No files match missing/); +}); + +test("executable adds executable bits without removing existing permissions", (t) => { + const directory = workspace(t); + for (const name of ["cli.js", "rpc.js"]) writeFileSync(join(directory, name), "entry", { mode: 0o640 }); + const mode = statSync(join(directory, "cli.js")).mode; + assert.equal(run(directory, "executable", "cli.js", "rpc.js").status, 0); + if (process.platform !== "win32") { + assert.equal(statSync(join(directory, "cli.js")).mode & 0o777, (mode | 0o111) & 0o777); + assert.equal(statSync(join(directory, "rpc.js")).mode & 0o111, 0o111); + } +}); + +test("invalid commands and missing arguments fail", (t) => { + const directory = workspace(t); + for (const args of [[], ["unknown", "dist"], ["clean"], ["copy", "dist"], ["executable"]]) { + const result = run(directory, ...args); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Usage: package-files/); + } +}); diff --git a/tsconfig.json b/tsconfig.json index eea4c716b..c032b98e2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,6 +9,20 @@ "@earendil-works/chord/context": ["./packages/chord/src/context/index.ts"], "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], "@earendil-works/chord/next": ["./packages/chord/src/next/index.ts"], + "@earendil-works/chord/next/plugins": ["./packages/chord/src/next-node/plugins/index.ts"], + "@earendil-works/chord/next/plugins/runtime": ["./packages/chord/src/next-node/plugins/runtime.ts"], + "@acme/app/*": ["./packages/chord/test/next/examples/49-plugin-api/app/*.ts"], + "@notes/app/*": ["./packages/chord/test/next/examples/53-app/app/*.ts"], + "@board/app/*": ["./packages/chord/test/next/examples/56-web/app/*.ts"], + "@chat/app/*": ["./packages/chord/test/next/examples/59-chat/app/*.ts"], + "@earendil-works/chord/next/http": ["./packages/chord/src/next-node/http/index.ts"], + "@earendil-works/chord/next/workers": ["./packages/chord/src/next-node/workers/index.ts"], + "@earendil-works/chord/next/app": ["./packages/chord/src/next-node/app/index.ts"], + "@earendil-works/chord/next/web": ["./packages/chord/src/next-web/index.ts"], + "@earendil-works/chord/next/web/server": ["./packages/chord/src/next-node/surfaces/index.ts"], + "@earendil-works/chord/next/react": ["./packages/chord/src/next-react/index.ts"], + "@earendil-works/chord/next/tui": ["./packages/chord/src/next-tui/index.ts"], + "@earendil-works/chord/next/tui/launcher": ["./packages/chord/src/next-node/tui/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"], @@ -24,6 +38,8 @@ "@earendil-works/pi-codemode/portable": ["./packages/codemode/src/portable.ts"], "@earendil-works/pi-codemode/source": ["./packages/codemode/src/source.ts"], "@earendil-works/pi-codemode/worker": ["./packages/codemode/src/runtime/worker.ts"], + "@earendil-works/pi-schema": ["./packages/schema/src/index.ts"], + "@earendil-works/pi-schema/formats": ["./packages/schema/src/formats/index.ts"], "@earendil-works/pi-mcp": ["./packages/mcp/src/index.ts"], "@earendil-works/pi-mcp/oauth": ["./packages/mcp/src/oauth/index.ts"], "@earendil-works/pi-mcp/testing": ["./packages/mcp/src/testing/index.ts"], @@ -51,5 +67,5 @@ } }, "include": ["scripts/model-catalog-protocol.ts", "scripts/model-catalog-protocol.test.ts", "packages/*/src/**/*", "packages/*/test/**/*", "packages/ai/catalog/**/*", "packages/durable/test-workers/**/*", "packages/coding-agent/examples/**/*", "packages/agent/examples/**/*"], - "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/gondolin/**"] + "exclude": ["**/dist/**", "packages/coding-agent/examples/extensions/**", "packages/coding-agent/test/suite/regressions/8261-subagent-project-trust.test.ts"] }