From 5bc41451654be22e79cc1a89ccc54a37f6d56691 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 2 Nov 2026 21:15:32 +0000 Subject: [PATCH] feat(chord): Chord v2 web and terminal surfaces (slice 11) - Web: base bundles built from the app's entry at start (content-hashed, served forever), plugin scripts that register themselves and one combined file, WebScripts/TuiScripts showing the current set, startPage with the connection loop, in-place plugin reloads and a reload on a new base bundle. - React bindings: HostProvider, useHold, useWatch, useRegistry, useRefState. - Terminal: startTerminal, a launcher that signs in with a token, a token file or device pairing and always runs the app's terminal code in a sandboxed child process (pipes, empty environment, --permission granting nothing, no SQLite, a builtin allowlist and a frozen process inside); output filter, confirmed copy/open/attach actions; startTerminalRuntime and TuiScope inside. - Themes: the Themes registry, ThemeChoice per principal, CSS variables on the page, terminal colours by role; CSS modules in plugin builds. - App kit: startServer gains web and tui; startApp renamed startServer, exportAddress/importAddress renamed exportData/importData and export no longer stops the worker; ephemeral sessions end with their client's last link. - Examples 56-web, 57-tui, 58-themes, 59-chat. --- package-lock.json | 65 +++ packages/chord/CHANGELOG.md | 3 +- packages/chord/docs/next/design.md | 8 +- .../docs/next/guide-plugins-hosts/book.md | 19 +- .../docs/next/guide-plugins-hosts/index.html | 19 +- packages/chord/docs/next/plugin-authoring.md | 12 + packages/chord/docs/next/slices.md | 26 +- packages/chord/package.json | 8 + packages/chord/src/next-node/app/README.md | 54 +- packages/chord/src/next-node/app/app.ts | 205 +++++--- packages/chord/src/next-node/app/archive.ts | 129 +++-- packages/chord/src/next-node/app/data.ts | 52 ++ .../src/next-node/app/ephemeral-sessions.ts | 80 +++ packages/chord/src/next-node/app/index.ts | 4 +- packages/chord/src/next-node/app/layout.ts | 4 +- packages/chord/src/next-node/app/sessions.ts | 45 +- packages/chord/src/next-node/app/worker.ts | 9 +- packages/chord/src/next-node/http/README.md | 7 +- packages/chord/src/next-node/http/headers.ts | 9 +- packages/chord/src/next-node/http/http.ts | 17 +- packages/chord/src/next-node/http/router.ts | 11 + packages/chord/src/next-node/http/upgrades.ts | 6 +- .../chord/src/next-node/plugins/README.md | 7 +- packages/chord/src/next-node/plugins/build.ts | 35 +- .../src/next-node/plugins/css-modules.ts | 65 +++ .../chord/src/next-node/plugins/evaluate.ts | 20 +- .../chord/src/next-node/plugins/runtime.ts | 155 +----- .../chord/src/next-node/surfaces/README.md | 40 ++ .../chord/src/next-node/surfaces/bundle.ts | 71 +++ .../chord/src/next-node/surfaces/current.ts | 70 +++ .../chord/src/next-node/surfaces/index.ts | 14 + .../chord/src/next-node/surfaces/published.ts | 159 ++++++ .../chord/src/next-node/surfaces/served.ts | 50 ++ packages/chord/src/next-node/surfaces/tui.ts | 67 +++ packages/chord/src/next-node/surfaces/web.ts | 137 +++++ packages/chord/src/next-node/tui/README.md | 75 +++ packages/chord/src/next-node/tui/actions.ts | 212 ++++++++ .../chord/src/next-node/tui/credentials.ts | 71 +++ packages/chord/src/next-node/tui/filter.ts | 75 +++ packages/chord/src/next-node/tui/index.ts | 8 + packages/chord/src/next-node/tui/launcher.ts | 466 ++++++++++++++++++ packages/chord/src/next-node/tui/pipe.ts | 132 +++++ packages/chord/src/next-node/tui/sandbox.ts | 450 +++++++++++++++++ packages/chord/src/next-react/README.md | 14 + packages/chord/src/next-react/hooks.ts | 79 +++ packages/chord/src/next-react/index.ts | 5 + packages/chord/src/next-react/stores.ts | 45 ++ packages/chord/src/next-tui/README.md | 29 ++ packages/chord/src/next-tui/bridge.ts | 69 +++ packages/chord/src/next-tui/channel.ts | 163 ++++++ packages/chord/src/next-tui/frames.ts | 51 ++ packages/chord/src/next-tui/index.ts | 14 + packages/chord/src/next-tui/runtime.ts | 155 ++++++ packages/chord/src/next-tui/scope.ts | 81 +++ packages/chord/src/next-tui/terminal.ts | 45 ++ packages/chord/src/next-tui/theme.ts | 44 ++ packages/chord/src/next-web/README.md | 25 + packages/chord/src/next-web/environment.ts | 37 ++ packages/chord/src/next-web/index.ts | 6 + packages/chord/src/next-web/page.ts | 85 ++++ packages/chord/src/next-web/scripts.ts | 79 +++ packages/chord/src/next-web/themes.ts | 38 ++ packages/chord/src/next/index.ts | 1 + packages/chord/src/next/link/values.ts | 14 +- packages/chord/src/next/surfaces/README.md | 37 ++ .../chord/src/next/surfaces/active-theme.ts | 63 +++ packages/chord/src/next/surfaces/client.ts | 216 ++++++++ .../chord/src/next/surfaces/connection.ts | 191 +++++++ packages/chord/src/next/surfaces/index.ts | 42 ++ packages/chord/src/next/surfaces/loaded.ts | 152 ++++++ .../chord/src/next/surfaces/registered.ts | 77 +++ packages/chord/src/next/surfaces/scripts.ts | 43 ++ .../chord/src/next/surfaces/theme-choice.ts | 82 +++ packages/chord/src/next/surfaces/themes.ts | 136 +++++ packages/chord/test/next-node/app/app.test.ts | 69 ++- .../chord/test/next-node/app/archive.test.ts | 26 +- .../next-node/app/ephemeral-sessions.test.ts | 51 ++ packages/chord/test/next-node/app/support.ts | 4 +- .../chord/test/next-node/fixtures/tui/api.ts | 7 + .../test/next-node/fixtures/tui/drive.ts | 17 + .../test/next-node/fixtures/tui/escapes.ts | 262 ++++++++++ .../chord/test/next-node/fixtures/tui/main.ts | 126 +++++ .../test/next-node/fixtures/tui/os-layer.ts | 95 ++++ .../next-node/fixtures/tui/sandbox-run.ts | 14 + .../chord/test/next-node/fixtures/web/api.ts | 14 + .../chord/test/next-node/fixtures/web/main.ts | 81 +++ .../chord/test/next-node/http/http.test.ts | 14 + .../test/next-node/http/upgrades.test.ts | 25 + .../test/next-node/plugins/build.test.ts | 59 +++ .../chord/test/next-node/react/hooks.test.ts | 45 ++ .../chord/test/next-node/react/stores.test.ts | 114 +++++ .../chord/test/next-node/surfaces/tui.test.ts | 56 +++ .../chord/test/next-node/surfaces/web.test.ts | 218 ++++++++ .../chord/test/next-node/tui/actions.test.ts | 228 +++++++++ .../test/next-node/tui/credentials.test.ts | 36 ++ .../chord/test/next-node/tui/filter.test.ts | 38 ++ .../chord/test/next-node/tui/launcher.test.ts | 263 ++++++++++ .../chord/test/next-node/tui/pipe.test.ts | 60 +++ .../chord/test/next-node/tui/sandbox.test.ts | 112 +++++ packages/chord/test/next-node/tui/support.ts | 116 +++++ packages/chord/test/next-node/web/browser.ts | 181 +++++++ .../chord/test/next-node/web/page.test.ts | 83 ++++ packages/chord/test/next-node/web/support.ts | 79 +++ .../chord/test/next/examples/53-app/main.ts | 2 +- .../chord/test/next/examples/53-app/start.ts | 6 +- .../test/next/examples/55-app-export/main.ts | 15 +- .../next/examples/56-web/app/principals.ts | 7 + .../chord/test/next/examples/56-web/app/v1.ts | 12 + .../test/next/examples/56-web/app/v1/tui.ts | 6 + .../test/next/examples/56-web/app/v1/web.ts | 10 + .../test/next/examples/56-web/headless.ts | 69 +++ .../chord/test/next/examples/56-web/main.ts | 58 +++ .../56-web/plugins/clock/package.json | 5 + .../next/examples/56-web/plugins/clock/web.ts | 6 + .../56-web/plugins/notes/package.json | 5 + .../next/examples/56-web/plugins/notes/tui.ts | 6 + .../next/examples/56-web/plugins/notes/web.ts | 6 + .../chord/test/next/examples/56-web/start.ts | 57 +++ .../test/next/examples/56-web/tui/main.ts | 30 ++ .../test/next/examples/56-web/web/main.ts | 33 ++ .../chord/test/next/examples/57-tui/main.ts | 68 +++ .../test/next/examples/57-tui/terminal.ts | 10 + .../test/next/examples/58-themes/main.ts | 124 +++++ .../58-themes/plugins/midnight.web.ts | 20 + .../58-themes/plugins/solarized/package.json | 5 + .../58-themes/plugins/solarized/theme.ts | 21 + .../58-themes/plugins/solarized/tui.ts | 6 + .../58-themes/plugins/solarized/web.ts | 6 + .../next/examples/59-chat/app/principals.ts | 8 + .../test/next/examples/59-chat/app/v1.ts | 73 +++ .../next/examples/59-chat/app/v1/server.ts | 10 + .../next/examples/59-chat/app/v1/worker.ts | 3 + .../chord/test/next/examples/59-chat/main.ts | 128 +++++ .../59-chat/plugins/chat/package.json | 5 + .../examples/59-chat/plugins/chat/server.ts | 45 ++ .../examples/59-chat/plugins/chat/worker.ts | 78 +++ .../examples/59-chat/plugins/login.server.ts | 5 + .../chord/test/next/examples/59-chat/start.ts | 76 +++ .../test/next/examples/59-chat/tui/main.ts | 90 ++++ .../test/next/examples/59-chat/web/main.ts | 95 ++++ .../test/next/examples/59-chat/web/style.css | 7 + .../test/next/examples/59-chat/worker.ts | 6 + packages/chord/test/next/link/values.test.ts | 8 + .../test/next/surfaces/active-theme.test.ts | 123 +++++ .../chord/test/next/surfaces/client.test.ts | 179 +++++++ .../test/next/surfaces/connection.test.ts | 214 ++++++++ .../test/next/surfaces/registered.test.ts | 57 +++ .../chord/test/next/surfaces/themes.test.ts | 70 +++ packages/chord/test/next/tui/frames.test.ts | 26 + packages/chord/test/next/tui/scope.test.ts | 80 +++ tsconfig.json | 7 + 151 files changed, 9376 insertions(+), 392 deletions(-) create mode 100644 packages/chord/src/next-node/app/data.ts create mode 100644 packages/chord/src/next-node/app/ephemeral-sessions.ts create mode 100644 packages/chord/src/next-node/plugins/css-modules.ts create mode 100644 packages/chord/src/next-node/surfaces/README.md create mode 100644 packages/chord/src/next-node/surfaces/bundle.ts create mode 100644 packages/chord/src/next-node/surfaces/current.ts create mode 100644 packages/chord/src/next-node/surfaces/index.ts create mode 100644 packages/chord/src/next-node/surfaces/published.ts create mode 100644 packages/chord/src/next-node/surfaces/served.ts create mode 100644 packages/chord/src/next-node/surfaces/tui.ts create mode 100644 packages/chord/src/next-node/surfaces/web.ts create mode 100644 packages/chord/src/next-node/tui/README.md create mode 100644 packages/chord/src/next-node/tui/actions.ts create mode 100644 packages/chord/src/next-node/tui/credentials.ts create mode 100644 packages/chord/src/next-node/tui/filter.ts create mode 100644 packages/chord/src/next-node/tui/index.ts create mode 100644 packages/chord/src/next-node/tui/launcher.ts create mode 100644 packages/chord/src/next-node/tui/pipe.ts create mode 100644 packages/chord/src/next-node/tui/sandbox.ts create mode 100644 packages/chord/src/next-react/README.md create mode 100644 packages/chord/src/next-react/hooks.ts create mode 100644 packages/chord/src/next-react/index.ts create mode 100644 packages/chord/src/next-react/stores.ts create mode 100644 packages/chord/src/next-tui/README.md create mode 100644 packages/chord/src/next-tui/bridge.ts create mode 100644 packages/chord/src/next-tui/channel.ts create mode 100644 packages/chord/src/next-tui/frames.ts create mode 100644 packages/chord/src/next-tui/index.ts create mode 100644 packages/chord/src/next-tui/runtime.ts create mode 100644 packages/chord/src/next-tui/scope.ts create mode 100644 packages/chord/src/next-tui/terminal.ts create mode 100644 packages/chord/src/next-tui/theme.ts create mode 100644 packages/chord/src/next-web/README.md create mode 100644 packages/chord/src/next-web/environment.ts create mode 100644 packages/chord/src/next-web/index.ts create mode 100644 packages/chord/src/next-web/page.ts create mode 100644 packages/chord/src/next-web/scripts.ts create mode 100644 packages/chord/src/next-web/themes.ts create mode 100644 packages/chord/src/next/surfaces/README.md create mode 100644 packages/chord/src/next/surfaces/active-theme.ts create mode 100644 packages/chord/src/next/surfaces/client.ts create mode 100644 packages/chord/src/next/surfaces/connection.ts create mode 100644 packages/chord/src/next/surfaces/index.ts create mode 100644 packages/chord/src/next/surfaces/loaded.ts create mode 100644 packages/chord/src/next/surfaces/registered.ts create mode 100644 packages/chord/src/next/surfaces/scripts.ts create mode 100644 packages/chord/src/next/surfaces/theme-choice.ts create mode 100644 packages/chord/src/next/surfaces/themes.ts create mode 100644 packages/chord/test/next-node/app/ephemeral-sessions.test.ts create mode 100644 packages/chord/test/next-node/fixtures/tui/api.ts create mode 100644 packages/chord/test/next-node/fixtures/tui/drive.ts create mode 100644 packages/chord/test/next-node/fixtures/tui/escapes.ts create mode 100644 packages/chord/test/next-node/fixtures/tui/main.ts create mode 100644 packages/chord/test/next-node/fixtures/tui/os-layer.ts create mode 100644 packages/chord/test/next-node/fixtures/tui/sandbox-run.ts create mode 100644 packages/chord/test/next-node/fixtures/web/api.ts create mode 100644 packages/chord/test/next-node/fixtures/web/main.ts create mode 100644 packages/chord/test/next-node/react/hooks.test.ts create mode 100644 packages/chord/test/next-node/react/stores.test.ts create mode 100644 packages/chord/test/next-node/surfaces/tui.test.ts create mode 100644 packages/chord/test/next-node/surfaces/web.test.ts create mode 100644 packages/chord/test/next-node/tui/actions.test.ts create mode 100644 packages/chord/test/next-node/tui/credentials.test.ts create mode 100644 packages/chord/test/next-node/tui/filter.test.ts create mode 100644 packages/chord/test/next-node/tui/launcher.test.ts create mode 100644 packages/chord/test/next-node/tui/pipe.test.ts create mode 100644 packages/chord/test/next-node/tui/sandbox.test.ts create mode 100644 packages/chord/test/next-node/tui/support.ts create mode 100644 packages/chord/test/next-node/web/browser.ts create mode 100644 packages/chord/test/next-node/web/page.test.ts create mode 100644 packages/chord/test/next-node/web/support.ts create mode 100644 packages/chord/test/next/examples/56-web/app/principals.ts create mode 100644 packages/chord/test/next/examples/56-web/app/v1.ts create mode 100644 packages/chord/test/next/examples/56-web/app/v1/tui.ts create mode 100644 packages/chord/test/next/examples/56-web/app/v1/web.ts create mode 100644 packages/chord/test/next/examples/56-web/headless.ts create mode 100644 packages/chord/test/next/examples/56-web/main.ts create mode 100644 packages/chord/test/next/examples/56-web/plugins/clock/package.json create mode 100644 packages/chord/test/next/examples/56-web/plugins/clock/web.ts create mode 100644 packages/chord/test/next/examples/56-web/plugins/notes/package.json create mode 100644 packages/chord/test/next/examples/56-web/plugins/notes/tui.ts create mode 100644 packages/chord/test/next/examples/56-web/plugins/notes/web.ts create mode 100644 packages/chord/test/next/examples/56-web/start.ts create mode 100644 packages/chord/test/next/examples/56-web/tui/main.ts create mode 100644 packages/chord/test/next/examples/56-web/web/main.ts create mode 100644 packages/chord/test/next/examples/57-tui/main.ts create mode 100644 packages/chord/test/next/examples/57-tui/terminal.ts create mode 100644 packages/chord/test/next/examples/58-themes/main.ts create mode 100644 packages/chord/test/next/examples/58-themes/plugins/midnight.web.ts create mode 100644 packages/chord/test/next/examples/58-themes/plugins/solarized/package.json create mode 100644 packages/chord/test/next/examples/58-themes/plugins/solarized/theme.ts create mode 100644 packages/chord/test/next/examples/58-themes/plugins/solarized/tui.ts create mode 100644 packages/chord/test/next/examples/58-themes/plugins/solarized/web.ts create mode 100644 packages/chord/test/next/examples/59-chat/app/principals.ts create mode 100644 packages/chord/test/next/examples/59-chat/app/v1.ts create mode 100644 packages/chord/test/next/examples/59-chat/app/v1/server.ts create mode 100644 packages/chord/test/next/examples/59-chat/app/v1/worker.ts create mode 100644 packages/chord/test/next/examples/59-chat/main.ts create mode 100644 packages/chord/test/next/examples/59-chat/plugins/chat/package.json create mode 100644 packages/chord/test/next/examples/59-chat/plugins/chat/server.ts create mode 100644 packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts create mode 100644 packages/chord/test/next/examples/59-chat/plugins/login.server.ts create mode 100644 packages/chord/test/next/examples/59-chat/start.ts create mode 100644 packages/chord/test/next/examples/59-chat/tui/main.ts create mode 100644 packages/chord/test/next/examples/59-chat/web/main.ts create mode 100644 packages/chord/test/next/examples/59-chat/web/style.css create mode 100644 packages/chord/test/next/examples/59-chat/worker.ts create mode 100644 packages/chord/test/next/surfaces/active-theme.test.ts create mode 100644 packages/chord/test/next/surfaces/client.test.ts create mode 100644 packages/chord/test/next/surfaces/connection.test.ts create mode 100644 packages/chord/test/next/surfaces/registered.test.ts create mode 100644 packages/chord/test/next/surfaces/themes.test.ts create mode 100644 packages/chord/test/next/tui/frames.test.ts create mode 100644 packages/chord/test/next/tui/scope.test.ts diff --git a/package-lock.json b/package-lock.json index a0749b93b..5655e8690 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2598,6 +2598,26 @@ "@types/retry": "*" } }, + "node_modules/@types/react": { + "version": "19.2.14", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", + "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", + "dev": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.3", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", + "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, "node_modules/@types/retry": { "version": "0.12.5", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", @@ -3431,6 +3451,13 @@ "node": ">= 8" } }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, "node_modules/data-uri-to-buffer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", @@ -4781,6 +4808,29 @@ "rc": "cli.js" } }, + "node_modules/react": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", + "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.3.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.3.0.tgz", + "integrity": "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "scheduler": "^0.28.0" + }, + "peerDependencies": { + "react": "^19.3.0" + } + }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -4875,6 +4925,13 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "license": "MIT" }, + "node_modules/scheduler": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.28.0.tgz", + "integrity": "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw==", + "dev": true, + "license": "MIT" + }, "node_modules/semver": { "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", @@ -5773,7 +5830,11 @@ "@earendil-works/pi-telemetry": "1.5.0" }, "devDependencies": { + "@types/react": "19.2.14", + "@types/react-dom": "19.2.3", "@types/ws": "8.18.2", + "react": "19.3.0", + "react-dom": "19.3.0", "vitest": "4.1.12", "ws": "8.21.0" }, @@ -5782,12 +5843,16 @@ }, "peerDependencies": { "esbuild": "0.28.2", + "react": "19.3.0", "ws": "8.21.0" }, "peerDependenciesMeta": { "esbuild": { "optional": true }, + "react": { + "optional": true + }, "ws": { "optional": true } diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index fb3520709..8ea8cb930 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -5,7 +5,8 @@ ### Added - Chord v2's plugins component gains the plugin model (slice 12): `createPlugins({ api: { name, versions } })` checks every import of the app's versioned API (`@acme/app/v3`, per-kind modules such as `@acme/app/v3/web`): an unlisted version, another kind's module, two versions in one facet and a direct import of Chord are build errors naming the file and line. Roots may be `{ path, required }`: a required root's plugin failing at `load` rejects it with a `PluginError` after the load settled. Files named `..` (`contract.web.tsx`) are of that kind, refused in facets of another kind with the chain of imports named. Single-file plugins (`hello.server.ts` at a root's top, grouped by name, host packages only). `installPlugin({ git, ref } | { npm, version }, root, ctx)` installs plugins from git (a shallow clone of a branch, tag or commit; a repository of several plugins too) or npm; a package lists the plugin folders it provides in its `package.json` (`"chord": { "plugins": ["./notes", "plugins/*", "!plugins/draft"] }`), used by discovery and `installPlugin` (without it: one plugin when the package's top has facet files, else its subfolders with a `package.json`). `testPlugin(dir, { kind, modules, stubs, principal }, ctx)` builds a plugin as production does and loads it on an in-memory host beside stub facets, collecting its contributions to registries nobody serves and listing the contracts it waits for. Agents' guide: `docs/next/plugin-authoring.md`. -- Chord v2 gains the app kit (slice 13, `src/next-node/app`, to be exported as `@earendil-works/chord/next/app`): `startApp

({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, install?, onError }, ctx)` lays out the data directory (`server/store.sqlite`, `workers//

/`, `plugins/`, `logs/`, `owner-token`; one server per data directory), creates the server's store and host, HTTP, one workers component per worker kind and the plugins component over the app's required root and the user's root, and loads the server's plugins with the kit's objects; `startWorker({ api, kinds, busy?, onError })` is a worker's entry (its directory and store, the link to the server first, its plugins). The start-data objects `AppInfo` and `WorkerInfo` serve process values to plugins. Sessions (`Sessions`: `issue`, `verify`, `revoke`, `list`; tokens kept as SHA-256 hashes, compared in constant time; persisted ones end after 400 days unused and their browser cookie is renewed daily while used, ephemeral ones live in memory, at most `limits.ephemeralSessions`) authenticate a signed `HttpOnly`, `SameSite=Strict` cookie or `Authorization: Bearer`; a one-time owner link per start (`app.ownerLink()`, the token in the URL's fragment, single use, 10 minutes), an owner token file (mode 0600, written atomically) and device pairing (`POST /_chord/pair`, owner approval, a poll returning the token once; bounded in all and per client address) sign the owner in. The data directory, its stores and secret files are made private (0700, 0600) at every start; the home directory, a file system's root and another user's directory are refused. `Files` per host: small and chunked writes at a stored offset, ranged reads, a watchable index, caps, at most `limits.uploads` uploads under way. `app.exportAddress(address, path, ctx)` and `app.importAddress(archive, address, ctx)` move a worker's store and files as a tar, holding the worker's process lock meanwhile (refusing an existing address, another app's archive, every path leaving the target, and archives past `limits.importBytes` or `limits.importEntries`). `app.applyChanges(ctx)` and `app.plugins.install(source, ctx)`. Optional plugins as exported facets: `defineUsers(profile)`, `inviteFacet()`, `ephemeralLoginFacet()`. +- Chord v2 gains the app kit (slice 13, `src/next-node/app`, to be exported as `@earendil-works/chord/next/app`): `startServer

({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, install?, onError }, ctx)` lays out the data directory (`server/store.sqlite`, `workers//

/`, `plugins/`, `logs/`, `owner-token`; one server per data directory), creates the server's store and host, HTTP, one workers component per worker kind and the plugins component over the app's required root and the user's root, and loads the server's plugins with the kit's objects; `startWorker({ api, kinds, busy?, onError })` is a worker's entry (its directory and store, the link to the server first, its plugins). The start-data objects `AppInfo` and `WorkerInfo` serve process values to plugins. Sessions (`Sessions`: `issue`, `verify`, `revoke`, `list`; tokens kept as SHA-256 hashes, compared in constant time; persisted ones end after 400 days unused and their browser cookie is renewed daily while used, ephemeral ones live in memory, at most `limits.ephemeralSessions`) authenticate a signed `HttpOnly`, `SameSite=Strict` cookie or `Authorization: Bearer`; a one-time owner link per start (`app.ownerLink()`, the token in the URL's fragment, single use, 10 minutes), an owner token file (mode 0600, written atomically) and device pairing (`POST /_chord/pair`, owner approval, a poll returning the token once; bounded in all and per client address) sign the owner in. The data directory, its stores and secret files are made private (0700, 0600) at every start; the home directory, a file system's root and another user's directory are refused. `Files` per host: small and chunked writes at a stored offset, ranged reads, a watchable index, caps, at most `limits.uploads` uploads under way. `app.exportData(address, path, ctx)` and `app.importData(archive, address, ctx)` move the data a worker created (its store and files) as a tar without stopping anything: a running worker writes its own store at one consistent point, else the server reads it under the worker's process lock (import refusing an existing address, another app's archive, every path leaving the target, and archives past `limits.importBytes` or `limits.importEntries`). `app.applyChanges(ctx)` and `app.plugins.install(source, ctx)`. Optional plugins as exported facets: `defineUsers(profile)`, `inviteFacet()`, `ephemeralLoginFacet()`. +- Chord v2 gains web and terminal surfaces (slice 11): `webSurface` and `tuiSurface` (`src/next-node/surfaces`) build a client kind's base bundle from the app's entry at start (content-hashed, served forever) and publish its plugins' facets as scripts that register themselves (`chordFacet("", "", function (require, module, exports) {…})`) and one combined file, with `WebScripts` and `TuiScripts` showing the current set (a rollout step publishes without waiting for clients) and the page served through the app's own template (`PageAssets`, `pageScripts`). `startPage` (`src/next-web`) runs a page's client host: the connection loop (`keepConnected`: 250 ms doubling to 5 s, wakes, silent sockets, `canRetry`), plugins applied in place on a new set (a failed one keeps its old facet), a reload when the server's base bundle changed, and the active theme as CSS variables. React bindings (`src/next-react`): `HostProvider`, `useHold`, `useWatch`, `useRegistry`, `useRefState` (React is an optional peer dependency). The terminal: `startTerminal` (`src/next-node/tui`) signs in with a token, a token file or device pairing and runs the app's terminal code always in a sandbox (a child process on pipes with Node's permission model: no file system, terminal, processes, workers, network or SQLite; Node 26.5.1 or later), filtering its output, relaying its link and confirming its actions (copy, open a URL, attach a file the user names); `startTerminalRuntime` and `TuiScope` (`src/next-tui`) inside it. Themes: the `Themes` registry, `ThemeChoice` (one dark and one light pick per principal), `followActiveTheme`, `themeVariables`, `terminalColor`. `startServer` gains `web` and `tui`. Plugin builds scope `*.module.css` files per plugin and file. Ephemeral sessions end with their client's last link (after a minute). The HTTP component's links endpoint gains an `opened` hook. - Chord v2's `HttpRequest` gains `remoteAddress`, the peer's address as the listener knows it (for limits per client). - Chord v2's HTTP component gains `readSignedCookie(request, name, secret)`: `readCookie`'s value with its signed expiry, for renewing a cookie before it expires. diff --git a/packages/chord/docs/next/design.md b/packages/chord/docs/next/design.md index da87d32a7..e48701e53 100644 --- a/packages/chord/docs/next/design.md +++ b/packages/chord/docs/next/design.md @@ -2272,11 +2272,11 @@ fails closed when stale; a start waits for a draining old holder without polling ### The app kit (`chord/next/app`; slice 13) -Decisions "The app kit, point 1" to "point 5". `startApp(options)` on the server, `startWorker(options)` in a worker +Decisions "The app kit, point 1" to "point 5". `startServer(options)` on the server, `startWorker(options)` in a worker entry. Opinionated and small: an app is its API, its built-in plugins and a start script. - **Shape:** the app's layout is fixed: `src/api/v/` (shared and per-kind modules) and the built-in plugins root - `plugins/`. The data directory is a `startApp` option (no environment variables), default a dot directory named after + `plugins/`. The data directory is a `startServer` option (no environment variables), default a dot directory named after the app in the home directory. One server; worker kinds are a list the app defines, each a host kind with its own params (sharding is the app's) and how it starts (entry, start data, principal, idle time); zero kinds is a server with clients only. @@ -2290,7 +2290,9 @@ entry. Opinionated and small: an app is its API, its built-in plugins and a star - **Storage:** one store for the server, a directory per worker under its kind and address with its store; plugins never use data-directory paths. A files plugin per address (`put`, chunked transfers, a watchable index, caps) is the default file storage; raw directories only where a real path is needed, owned by one object, under the worker's - directory. Export and import per address: the store's archive plus the worker's directory. + directory. Export and import the data a host created (`exportData`, `importData`): the store's archive plus its + files; export stops nothing (a running worker packs its own store; without one the server reads it under the + worker's lock), import refuses an address that has data (decision "App kit naming and export"). - **Plugins:** the app's root (required) and the user's root in the data directory; installs from git or npm; one "apply changes" call builds and rolls out to the server, then every running worker, exposed by the app as it likes. diff --git a/packages/chord/docs/next/guide-plugins-hosts/book.md b/packages/chord/docs/next/guide-plugins-hosts/book.md index ad2d511f2..5f9576170 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/book.md +++ b/packages/chord/docs/next/guide-plugins-hosts/book.md @@ -268,7 +268,7 @@ a `remote` object is made per principal, so a guest can get a read-only variant :::note | The app kit does this wiring Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components (`createWorkers`, `chord.http`). An app built with the app kit (`chord/next/app`, the last chapter) does not wire them: -`startApp(...)` on the server and `startWorker(...)` in a worker's entry do. +`startServer(...)` on the server and `startWorker(...)` in a worker's entry do. ::: # Client Hosts {#clients lede="Browser tabs and terminal UIs are hosts too: what they can do, and what they cannot."} @@ -633,7 +633,7 @@ same name, the first root wins, and the other is reported. as production does, loads it on an in-memory host beside stub facets serving what it uses, collects what it contributes to registries nobody serves, and lists the contracts it waits for (`docs/next/plugin-authoring.md`). -# The App Kit {#kit lede="startApp and startWorker: the hosts, stores, sign-in, files and plugin roots every app needs, so an app is its API, its plugins and a start script."} +# The App Kit {#kit lede="startServer and startWorker: the hosts, stores, sign-in, files and plugin roots every app needs, so an app is its API, its plugins and a start script."} The chapters so far wired an app by hand: a store, a host, `createHttp`, `createWorkers`, `createPlugins`, an authenticator, a worker entry connecting to its parent. Every app needs the same wiring, and most of it has one right @@ -646,7 +646,7 @@ The notes app of example 53 is four files of base code and two plugins: ```ts // start.ts -const app = await startApp({ +const app = await startServer({ name: "notes", // the data directory: ~/.notes unless dataDir says otherwise root: "./plugins", // the app's own plugins: a required root api: { name: "@notes/app", versions: ["v1"], modules: { "@notes/app/v1": v1, "@notes/app/v1/server": v1server } }, @@ -666,7 +666,7 @@ console.log(`Open ${app.ownerLink()}`); startWorker({ api: API, kinds: KINDS, onError: (error) => log(error) }); ``` -`startApp` creates the data directory (`server/store.sqlite`, `workers//
/`, `plugins/`, `logs/`, +`startServer` creates the data directory (`server/store.sqlite`, `workers//
/`, `plugins/`, `logs/`, `owner-token`), opens the server's store, creates the server host, HTTP, a workers component per worker kind and the plugins component over two roots: the app's `plugins/` and the user's `/plugins/`. Then it loads the server's plugins with the kit's own facets beside them and resolves. `startWorker` does the worker's half: the project's @@ -707,9 +707,10 @@ Users other than the owner come from optional plugins the app adds: `defineUsers The kit never reloads by itself. An edit or an install marks plugins dirty; `app.plugins.state` drives the app's "apply changes" button, and `app.applyChanges(ctx)` builds and rolls out to the server, then the workers. -A project's data is its directory. `app.exportAddress(Project({ id }), path, ctx)` stops its worker and writes one -archive (a manifest, the store's export, the files); `app.importAddress(path, Project({ id: "copy" }), ctx)` makes a new -project from it and refuses one that already has data (example 55). +A project's data is what its worker created: its store and its files. `app.exportData(Project({ id }), path, ctx)` +writes them as one archive (a manifest, the store's export, the files) and stops nothing: a running worker writes its +own store at one consistent point. `app.importData(path, Project({ id: "copy" }), ctx)` makes a new project from it and +refuses one that already has data (example 55). # Reference {.appendix} @@ -725,8 +726,8 @@ project from it and refuses one that already has data (example 55). | `plugins.load(host, { kind, core, modules }, ctx)`, `build(names)`, `rollout(steps, built)` | load, build, roll out | | `plugins.installPlugin(source, root, ctx)`, `testPlugin(dir, options, ctx)` | install a plugin, test one alone | | `hostKind(name, params?)`, `Kind(params)` | host kinds and addresses | -| `startApp

({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, onError }, ctx)` | an app's server, with the kit | +| `startServer

({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, onError }, ctx)` | an app's server, with the kit | | `startWorker({ api, kinds, busy?, onError })` | a worker's entry, with the kit | | `app.ownerLink()`, `app.applyChanges(ctx)`, `app.plugins.install(source, ctx)` | sign in, apply changes, install | -| `app.exportAddress(address, path, ctx)`, `app.importAddress(path, address, ctx)` | move a worker's data | +| `app.exportData(address, path, ctx)`, `app.importData(path, address, ctx)` | move a worker's data | | `AppInfo`, `WorkerInfo`, `Files`, `Sessions` | the kit's objects plugins get | diff --git a/packages/chord/docs/next/guide-plugins-hosts/index.html b/packages/chord/docs/next/guide-plugins-hosts/index.html index beac6027f..d0ec02a3c 100644 --- a/packages/chord/docs/next/guide-plugins-hosts/index.html +++ b/packages/chord/docs/next/guide-plugins-hosts/index.html @@ -309,7 +309,7 @@ tab.connect(webSocketLink(remote object is made per principal, so a guest can get a read-only variant of the same notes.

The app kit does this wiring

Starting workers, routing a host kind to them, the WebSocket entry and authentication are slice 10's components (createWorkers, chord.http). An app built with the app kit (chord/next/app, the last chapter) does not wire them: -startApp(...) on the server and startWorker(...) in a worker's entry do.

+startServer(...) on the server and startWorker(...) in a worker's entry do.

@@ -594,7 +594,7 @@ contributes to registries nobody serves, and lists the contracts it waits for (< -
X

The App Kit

startApp and startWorker: the hosts, stores, sign-in, files and plugin roots every app needs, so an app is its API, its plugins and a start script.

+
X

The App Kit

startServer and startWorker: the hosts, stores, sign-in, files and plugin roots every app needs, so an app is its API, its plugins and a start script.

The chapters so far wired an app by hand: a store, a host, createHttp, createWorkers, createPlugins, an authenticator, a worker entry connecting to its parent. Every app needs the same wiring, and most of it has one right answer. The app kit (chord/next/app) is that answer, opinionated on purpose: one data directory layout, one way @@ -602,7 +602,7 @@ to sign in, one place for files.

A whole app

The notes app of example 53 is four files of base code and two plugins:

// start.ts
-const app = await startApp<NotesPrincipal>({
+const app = await startServer<NotesPrincipal>({
 	name: "notes",                                  // the data directory: ~/.notes unless dataDir says otherwise
 	root: "./plugins",                              // the app's own plugins: a required root
 	api: { name: "@notes/app", versions: ["v1"], modules: { "@notes/app/v1": v1, "@notes/app/v1/server": v1server } },
@@ -620,7 +620,7 @@ console.log(`Open ${app.ownerLink()
 
 // worker.ts
 startWorker({ api: API, kinds: KINDS, onError: (error) => log(error) });
-

startApp creates the data directory (server/store.sqlite, workers/<kind>/<address>/, plugins/, logs/, +

startServer creates the data directory (server/store.sqlite, workers/<kind>/<address>/, plugins/, logs/, owner-token), opens the server's store, creates the server host, HTTP, a workers component per worker kind and the plugins component over two roots: the app's plugins/ and the user's <dataDir>/plugins/. Then it loads the server's plugins with the kit's own facets beside them and resolves. startWorker does the worker's half: the project's @@ -652,9 +652,10 @@ returns its token (example 54).

Applying changes, moving projects

The kit never reloads by itself. An edit or an install marks plugins dirty; app.plugins.state drives the app's "apply changes" button, and app.applyChanges(ctx) builds and rolls out to the server, then the workers.

-

A project's data is its directory. app.exportAddress(Project({ id }), path, ctx) stops its worker and writes one -archive (a manifest, the store's export, the files); app.importAddress(path, Project({ id: "copy" }), ctx) makes a new -project from it and refuses one that already has data (example 55).

+

A project's data is what its worker created: its store and its files. app.exportData(Project({ id }), path, ctx) +writes them as one archive (a manifest, the store's export, the files) and stops nothing: a running worker writes its +own store at one consistent point. app.importData(path, Project({ id: "copy" }), ctx) makes a new project from it and +refuses one that already has data (example 55).

A

Reference

@@ -706,7 +707,7 @@ project from it and refuses one that already has data (example 55).

host kinds and addresses -startApp<P>({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, onError }, ctx) +startServer<P>({ name, dataDir?, root?, listen?, api, kinds, principal, workers?, files?, onError }, ctx) an app's server, with the kit @@ -718,7 +719,7 @@ project from it and refuses one that already has data (example 55).

sign in, apply changes, install -app.exportAddress(address, path, ctx), app.importAddress(path, address, ctx) +app.exportData(address, path, ctx), app.importData(path, address, ctx) move a worker's data diff --git a/packages/chord/docs/next/plugin-authoring.md b/packages/chord/docs/next/plugin-authoring.md index c21f477a3..64c2063b4 100644 --- a/packages/chord/docs/next/plugin-authoring.md +++ b/packages/chord/docs/next/plugin-authoring.md @@ -114,6 +114,18 @@ No type check, no build hooks, no `dist` needed; generated files are committed. process of its kind with that process's API modules. A failed build or evaluation leaves the plugin out (at startup) or keeps its old code (at a reload), reported with the error. +## Styles and themes (web and terminal facets) + +- A web facet's CSS is served with its script and swapped when the plugin reloads. Scope it with a **CSS module**: a + file named `*.module.css` has its class names made unique to the plugin and the file, and its import is the mapping + (`import styles from "./notes.module.css";
`). Plain CSS files still work; their class + names should then start with the plugin's name. +- Read the app's look from its CSS variables (`var(--accent)`, `var(--bg)`, `var(--fg)`, `var(--muted)`, `var(--error)`, + `var(--warning)`, `var(--success)`, and the app's own roles), never fixed colours: the active theme sets them. +- In a terminal facet, ask the app's theme lookup for a role (`theme.fg("accent", "Notes")`); never write raw colours. +- A theme is data: contribute a `Theme` (`{ id, name, mode, colors, web?, tui? }`) to the kit's `Themes` registry, + from a web facet and a terminal facet to reach both. Users pick one dark and one light theme. + ## Installing - `plugins.installPlugin({ git: "https://github.com/bob/stats.git", ref: "v1.2.0" }, root, ctx)`: a shallow clone of diff --git a/packages/chord/docs/next/slices.md b/packages/chord/docs/next/slices.md index 808044528..33a7c8ffe 100644 --- a/packages/chord/docs/next/slices.md +++ b/packages/chord/docs/next/slices.md @@ -37,6 +37,13 @@ slice spec's section citations are for orientation, not a reading limit (owner, - **Slice 10** (built 2026-10-31, `slice-10.md`): `src/next-node/http/` (a `fetch` core with the security baseline, Chord's Node listener, `nodeRequest`, plugin sockets, links), `src/next-node/workers/`, `createPluginRuntime`, kinds, the kernel's `RouteHandle` and anonymous `host.root`, origins only from relays; examples 40-48. +- **Slice 12** (committed 2026-11-01, `slice-12.md`): the plugin model in `src/next-node/plugins/` (the app's versioned + API, per-kind files, single-file plugins, installs from git and npm, `testPlugin`); examples 49-52. +- **Slice 13** (committed 2026-11-02, `slice-13.md`): the app kit, `src/next-node/app/` (`startServer`, `startWorker`, + sessions, owner link, pairing, `Files`, `exportData`/`importData`); examples 53-55. +- **Slice 11** (committed 2026-11-03, `slice-11.md`): `src/next/surfaces/` (scripts objects, registered scripts, the + connection loop, clients, themes), `src/next-web/`, `src/next-react/`, `src/next-tui/`, `src/next-node/surfaces/`, + `src/next-node/tui/` (the launcher and its child-process sandbox); examples 56-59. - **Chord's shared code** (upstream pi): `src/watch.ts` (`Watch`, `NativeWatch`, `WatchHandle`, `WatchEnd`, `NativeWatchable`), Delta (`track`, `diffRevisions`, `applyImmutable`, encoders, `PendingOps`, `appendText`), context, JSON, output, SQLite drivers. @@ -64,7 +71,7 @@ telemetry last, since it instruments the app kit too; 2026-11-01: the plugin mod | 7c | **Plugins** | `chord/next/plugins` (discovery, `buildFacet`, `evaluateFacet`, `pluginFile`, `installDependencies`, the watcher, `rollout`, `mapStack`; no dev/prod split; the kernel's `load`/`reload` per plugin; `slice-7c.md`) | | 10 | **HTTP and workers** | `chord/next/http` (server, `Http` registry, responses as descriptions, WebSocket entry) and `chord/next/workers` (a worker per address, IPC, idle stop) | | 12 | **The plugin model** | imports (relative within a repository, package names pinned across), versioned app API with Chord re-exported, per-kind API modules and contract files with the build's import check, single-file plugins, git installs, `testPlugin`, a required root (`slice-12.md`) | -| 13 | **The app kit** | `chord/next/app`: `startApp`, `startWorker`, layout, worker kinds, owner link, deny by default, sessions, CLI tokens, storage, the files plugin, export per address, plugin roots and "apply changes", base code as API, bootstrap and a start-data facet (`slice-13.md`) | +| 13 | **The app kit** | `chord/next/app`: `startServer`, `startWorker`, layout, worker kinds, owner link, deny by default, sessions, CLI tokens, storage, the files plugin, export per address, plugin roots and "apply changes", base code as API, bootstrap and a start-data facet (`slice-13.md`) | | 11 | **Web and TUI** | `chord/next/web` (page host, React hooks) and `chord/next/tui` (launcher, sandbox), from p2's code | | 9 | **Telemetry** | spans, events, metrics across hosts | @@ -147,7 +154,8 @@ pattern, no code. Decided cuts are noted where they apply. | 22 | relays, size limits per part, compression at the origin, adapted bindings get diffs | 4a (relays), 4b (limits, compression), 7b | | 22 | trace carriers (R5) | 9 | | 23 | HTTP (routes, wraps, responses as descriptions, request bodies, plugin sockets), workers; channels and chunked transfers stay owner patterns until a second user | 10 | -| 23 | the app kit: `startApp`, `startWorker`, start-data objects, deny by default, sessions, owner link and token file, device pairing, files (with chunked transfers), export and import per address, plugin roots and "apply changes", optional users, invite and ephemeral-login plugins | 13 | +| 23 | the app kit: `startServer`, `startWorker`, start-data objects, deny by default, sessions, owner link and token file, device pairing, files (with chunked transfers), export and import per address, plugin roots and "apply changes", optional users, invite and ephemeral-login plugins | 13 | +| 23 | web and TUI: base bundles, registered plugin scripts and the combined file, `WebScripts`/`TuiScripts`, `startPage` with the connection loop and in-place reloads, React bindings, the sandboxed terminal (launcher, sandbox, `TuiScope`), themes (`Themes`, `ThemeChoice`, CSS variables, terminal colours), CSS modules | 11 | | 24 | Durable integration: `chord(fn, context)` (Q6), a `RegistryReader` over composed registries; the harness object and its model and env registries are p3's (base code, owner, 2026-10-27) | 8 (Chord parts), p3 | | 24 | `StoreStorage` | 6b | | 27 | versions table: contracts (L1 done), registries 8, documents 6, jobs 5, link protocol 3a | as listed | @@ -166,12 +174,6 @@ change; a slice's spec includes every entry of its section, and its build and re deleted (the slice's spec and code are the record). Moved here from `carry-over.md` (owner, 2026-10-24). -### Slice 11 (with the app kit's follow-ups) - -- **App kit renames and export** (decision "App kit naming and export"): `startApp` → `startServer`; - `exportAddress`/`importAddress` → `exportData`/`importData`; export without stopping the worker (a kit object on the - worker exports its store and finished files; without a worker the server takes the lock and reads the store); import - unchanged apart from the name. Examples 53-55, the app README and the guide's app kit chapter follow. ### Any slice (protocol and checks) - **Bump `PROTOCOL_VERSION`** for any message or reason change once p3 has shipped (slice 3b added `retry`, `loading` @@ -191,14 +193,6 @@ deleted (the slice's spec and code are the record). Moved here from `carry-over. - **`rebuild` of tables** (key and column-type changes): `evolveTable` has no edit for them; build one only when a real case needs it. (Slice 6a's second spec review; out of slice 7b's scope, `slice-7b.md` §12.) -### Slice 11: web and TUI - -- **Ephemeral sessions end with the connection** (slice 13 §3): the kit's ephemeral login drops its sessions when the - server starts and gives the browser a cookie without a lifetime; ending a session when its client's last link - closes needs the clients' connection loop, which slice 11 builds. (Slice 13 build.) -- **`startApp` gains the web surface** (slice 13 §6): serving the clients' bundles from the kit, behind its sessions. - (Slice 13 spec.) - ### After slice 11 - **Messages and languages in the app kit** (decision "Messages and languages in the app kit", proposal diff --git a/packages/chord/package.json b/packages/chord/package.json index 6e8e12a4d..c61aef777 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -95,18 +95,26 @@ }, "peerDependencies": { "esbuild": "0.28.2", + "react": "19.3.0", "ws": "8.21.0" }, "peerDependenciesMeta": { "esbuild": { "optional": true }, + "react": { + "optional": true + }, "ws": { "optional": true } }, "devDependencies": { + "@types/react": "19.2.14", + "@types/react-dom": "19.2.3", "@types/ws": "8.18.2", + "react": "19.3.0", + "react-dom": "19.3.0", "vitest": "4.1.12", "ws": "8.21.0" } diff --git a/packages/chord/src/next-node/app/README.md b/packages/chord/src/next-node/app/README.md index 7fcd9e4a0..254e5da54 100644 --- a/packages/chord/src/next-node/app/README.md +++ b/packages/chord/src/next-node/app/README.md @@ -1,7 +1,7 @@ # app The app kit (slice 13, design §23 "The app kit", decisions "The app kit, point 1" to "point 5"): an opinionated layer -over `http`, `workers` and `plugins` so an app is its API, its built-in plugins and a start script. `startApp` on the +over `http`, `workers` and `plugins` so an app is its API, its built-in plugins and a start script. `startServer` on the server, `startWorker` in a worker's entry; the kit makes the data directory, the stores, the hosts, HTTP with deny by default, sessions and the owner's sign-in, device pairing, each host's files, export and import per address, and the plugin roots with "apply changes". Node-only. Exported as `@earendil-works/chord/next/app` once v2 has its entry @@ -11,7 +11,7 @@ point. Examples 53 (`53-app/`, a whole small app), 54 (sessions and pairing) and ```ts // start.ts (the app's start script) -const app = await startApp

({ +const app = await startServer

({ name: "acme", // the default data directory: ~/.acme dataDir: process.argv[2], // optional; no environment variables root: "./plugins", // the app's built-in plugins (required root); this is the default @@ -31,14 +31,16 @@ const app = await startApp

({ install: { npmCommand: ["npm"] }, // how installs run npm limits: { ephemeralSessions: 1000, pendingPairings: 100, pairingsPerAddress: 10, // the defaults uploads: 64, importBytes: 4 * 2 ** 30, importEntries: 100_000 }, + web: { entry: "./src/web/main.tsx", page: (request, assets) => shell(request.nonce, assets) }, // slice 11 + tui: { entry: "./src/tui/main.ts" }, onError: (error) => log(error), // AppError: { kind: "host" | "http" | "workers" | "plugins", error } }, ctx); console.log(`Open ${app.ownerLink()}`); app.host; app.http; app.workers.get("project"); app.plugins.state; app.dataDir; app.ownerTokenFile; await app.plugins.install({ git: url, ref: "v1" }, ctx); // into /plugins/ await app.applyChanges(ctx); // build the dirty plugins, roll out: server, then workers -await app.exportAddress(Project({ id }), "p.tar", ctx); -await app.importAddress("p.tar", Project({ id: "copy" }), ctx); +await app.exportData(Project({ id }), "p.tar", ctx); +await app.importData("p.tar", Project({ id: "copy" }), ctx); await app.close(ctx); // http, workers, host, store, plugins, the lock // worker.ts @@ -50,7 +52,7 @@ startWorker

({ api, kinds, busy, onError }); - **The API's modules** are named by convention: `/` for every kind and `//` for a kind's own; a kind's `packages` (React for a web kind) are host packages besides. The modules are this process's instances; the worker's entry passes the same `api` and `kinds`. -- **`startApp`**, in order: the data directory, by its real path (refused when it is the home directory, a file +- **`startServer`**, in order: the data directory, by its real path (refused when it is the home directory, a file system's root, or another user's: the kit would make it private), its lock (one server per data directory, also through a symbolic link: a second is refused after a second's wait), the layout made private (below), the cookie secret, the server's store, the host (address `Server()`, `principal.server`), `createHttp` (the links endpoint at `/chord`), the plugins component with the two roots, one @@ -94,9 +96,12 @@ and reads `AppInfo`. an ephemeral one only while the browser runs); a malformed token or an id that is no session (a prototype key included) is a plain 401. Browser and device sessions persist in the server's store; `lastUsed` is written at most once a minute, and a session unused for 400 days ends (checked when it is used, removed then). Ephemeral - sessions (a public login) live in memory only, never listed: at most `limits.ephemeralSessions`, none outliving the - server's run, so a login costs the same whatever came before. Past the cap the oldest ends: a flood of logins signs - out the earliest ephemeral users (the trade for bounded memory). + sessions (a public login) live in memory only (`ephemeral-sessions.ts`), never listed: at most + `limits.ephemeralSessions`, none outliving the server's run, so a login costs the same whatever came before. Past + the cap the oldest ends: a flood of logins signs out the earliest ephemeral users (the trade for bounded memory). An + ephemeral session ends with its client's connection (slice 11): once the last Chord link opened with it closed and + none opened for a minute (a reload, a reconnect), or when none opened within a minute of the login; the links + endpoint's `opened` hook counts them. - **The owner link:** each start makes a one-time token (single use, 10 minutes, the previous start's no longer valid), `app.ownerLink()` is `http://:/_chord/owner#` (`127.0.0.1` for `0.0.0.0`, `[::1]` for `::`, an IPv6 address in brackets). The page at `GET /_chord/owner` posts @@ -170,13 +175,17 @@ plugin's own remote methods, which call it. ## Export and import -`exportAddress(address, path, ctx)` stops the address's worker (`workers.stop`) and takes its process lock (the one a -worker takes at start) for the whole pack, writes the archive to `path` (mode 0600, staged and renamed), then releases -the lock and lets the address start again (`workers.retry`): a hold arriving meanwhile starts a worker that waits for -the lock, then opens the unchanged state. `importAddress` holds the same lock while it unpacks. A failed address (a -crashed worker nobody retried) is not stopped or retried: it stays failed. The archive is a ustar of +`exportData(address, path, ctx)` writes the data the worker at `address` created (its store and finished files) to +`path` (mode 0600, staged and renamed), stopping nothing (decision "App kit naming and export"): a running worker packs +its own open store (`HostData`, `data.ts`, a kit object only the server's principal may use; the store's export holds +the commit queue while it reads, so the archive is one consistent point) and its files; with no worker running, the +server takes the worker's process lock (no worker starts meanwhile) and packs the store itself. A file added or removed +while the export runs may be outside that point (the index is the store's): each file is opened when the archive +comes to it (one descriptor at a time), one deleted meanwhile skipped or read from its open handle. `importData` takes the same lock (held: a +worker runs, so the address has data: refused) and changes nothing else. Registering an imported host in the app's own +index is the app's. The archive is a ustar of `manifest.json` (`{ format: 1, app, api, address }`), `store.archive` (the store's export, JSONL) and `files/…`. -`importAddress(archive, address, ctx)` refuses an address that has a directory, unpacks beside it and renames into +`importData(archive, address, ctx)` refuses an address that has a directory, unpacks beside it and renames into place: the manifest must come first and name this app and format 1; only regular files and directories under `manifest.json`, `store.archive` and `files/` are taken; absolute paths, `..`, `.`, empty segments, backslashes, NULs, links, devices and bad checksums are refused (the kit's own tar reader, `tar.ts`); past `limits.importBytes` unpacked @@ -185,14 +194,25 @@ archive leaves nothing. The next hold starts the worker on the imported store an ## Plugins -Roots: the app's `root` (required: a plugin of it failing stops `startApp`) and `/plugins/` (the user's: +Roots: the app's `root` (required: a plugin of it failing stops `startServer`) and `/plugins/` (the user's: installs go there). `app.plugins.state` is the plugins component's marks for the app's "apply changes" button; `applyChanges` builds the dirty plugins (and installed ones) and rolls out to the server, then each worker kind; nothing is applied by itself. ## Internals -`app.ts` (`startApp`, the `App`), `worker.ts` (`startWorker`), `layout.ts` (the layout, private directories and +`app.ts` (`startServer`, the `App`), `worker.ts` (`startWorker`), `layout.ts` (the layout, private directories and stores), `secrets.ts` (tokens, hashes, files of mode 0600), `sessions.ts`, `owner.ts` (owner link and token file), `pairing.ts`, `files.ts` and `uploads.ts` (chunked uploads), `info.ts` (start-data objects), `archive.ts` and `tar.ts` -(export and import), and the optional plugins `users.ts`, `invite.ts`, `ephemeral.ts`. +(export and import), `ephemeral-sessions.ts`, and the optional plugins `users.ts`, `invite.ts`, `ephemeral.ts`. + +## Web and terminal (slice 11) + +With `web` the kit builds the page's base bundle from `web.entry`, publishes the `web` kind's plugin scripts and serves +the page through `web.page` at `web.paths` (`["/"]`), signed in unless `web.public` (`webSurface`, +`next-node/surfaces`); with `tui` the terminal's base bundle and the `tui` kind's scripts for launchers +(`tuiSurface`). Plugin assets are served under `/_chord/web/assets/` (the plugins component's `publicPath`). +`applyChanges` rolls out to the server, the workers, then the surfaces (`web`, `tui`: the new set published, clients +not waited for). `app.web` and `app.tui` give the base bundles' hashes. The server also serves the `Themes` registry and +`ThemeChoice` (each principal's dark and light pick, kept in its store). Without `web` and `tui` an app has no client +surface (API only). diff --git a/packages/chord/src/next-node/app/app.ts b/packages/chord/src/next-node/app/app.ts index f9796714d..5879dd652 100644 --- a/packages/chord/src/next-node/app/app.ts +++ b/packages/chord/src/next-node/app/app.ts @@ -1,4 +1,4 @@ -// `startApp` (slice 13 §1, §5; decisions "The app kit, point 1" to "point 5"): the server of an app built with the kit. +// `startServer` (slice 13 §1, §5; decisions "The app kit, point 1" to "point 5"): the server of an app built with the kit. // It lays out the data directory, opens the server's store, creates the server host, the HTTP component and one // workers component per worker kind, the plugins component over the app's required root and the user's root, loads // the server's plugins with the kit's own facets (the start-data object, sessions, the owner link, pairing, files), @@ -9,8 +9,10 @@ import { type Context, withAbortSignal } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; import { type Address, type HostKind, hostKind, type KindShape } from "../../next/contracts/index.ts"; import type { Principal } from "../../next/data/index.ts"; -import { createHost, type Host, type HostError, type PluginFacet } from "../../next/host/index.ts"; +import { canonicalJson } from "../../next/data/principal.ts"; +import { createHost, type Facet, type Host, type HostError, type PluginFacet } from "../../next/host/index.ts"; import { openStore, type Store, sqliteBackend } from "../../next/store/index.ts"; +import { themeChoiceFacet, themesFacet } from "../../next/surfaces/index.ts"; import { openNodeSqliteDatabase } from "../../sqlite/node.ts"; import type { Watchable } from "../../watch.ts"; import { createHttp, type HttpComponent, type HttpError, type ListenAddress } from "../http/index.ts"; @@ -23,16 +25,19 @@ import { type PluginsError, type PluginsState, type RolloutResult, + type RolloutStep, } from "../plugins/index.ts"; +import { tuiSurface, WEB_PREFIX, type WebSurfaceOptions, webSurface } from "../surfaces/index.ts"; import { createWorkers, type ProcessLock, processLock, type Workers, type WorkersError } from "../workers/index.ts"; -import { ARCHIVE_FORMAT, type ArchiveManifest, packAddress, unpackAddress } from "./archive.ts"; +import { ARCHIVE_FORMAT, type ArchiveManifest, packData, unpackData } from "./archive.ts"; +import { addressParam, HostData } from "./data.ts"; import { type FilesCaps, filesFacet } from "./files.ts"; import { appInfoFacet } from "./info.ts"; import { type AppLayout, createLayout, defaultDataDir, openDataDir, privateStore, workerDirectory } from "./layout.ts"; import { OWNER_PATH, OwnerLink, type OwnerPrincipal, ownerFacet, ownerTokenFile } from "./owner.ts"; import { Pairings, pairingFacet } from "./pairing.ts"; import { secretFile } from "./secrets.ts"; -import { sessionsFacet } from "./sessions.ts"; +import { createSessions } from "./sessions.ts"; /** The server's host kind: the kit's server host is `Server()`. */ export const Server = hostKind("server"); @@ -78,7 +83,7 @@ export type AppError = | { readonly kind: "workers"; readonly error: WorkersError } | { readonly kind: "plugins"; readonly error: PluginsError }; -/** What `startApp` takes. `{ kind: "owner" }` must be one of the app's principals (the kit's owner). */ +/** What `startServer` takes. `{ kind: "owner" }` must be one of the app's principals (the kit's owner). */ export type AppOptions

= { /** The app's name: its default data directory is `~/.`, its archives carry it. */ readonly name: string; @@ -101,6 +106,13 @@ export type AppOptions

= { readonly limits?: AppLimits; /** How installs run npm (slice 12's `InstallOptions`): `npm` on the `PATH` by default. */ readonly install?: InstallOptions; + /** + * The web surface (slice 11): the app's web entry (the base bundle's), its page template, the paths serving the + * page (`["/"]`), whether the page is public; the `web` facet kind. None: no web surface. + */ + readonly web?: AppWeb; + /** The terminal surface (slice 11): the app's terminal entry; the `tui` facet kind. None: no terminal clients. */ + readonly tui?: { readonly entry: string }; readonly onError: (error: AppError) => void; } & (OwnerPrincipal extends P ? unknown : { readonly "the app's principal type must have { kind: 'owner' }": never }); @@ -119,6 +131,9 @@ export type AppLimits = { readonly importEntries?: number; }; +/** The kit's web surface options (`webSurface`'s, its kind `web` and plugins the kit's). */ +export type AppWeb = Pick; + /** The plugins of a running app, for its "apply changes" control and its installs. */ export type AppPlugins = { /** The marks (`dirty`, `install`, `set`, `base`): what the app's button shows. */ @@ -136,18 +151,29 @@ export interface App

{ /** The workers components, by host kind name. */ readonly workers: ReadonlyMap>; readonly plugins: AppPlugins; + /** The web surface's base bundle hash, when the app has one. */ + readonly web: { readonly base: string } | undefined; + /** The terminal surface's base bundle hash, when the app has one. */ + readonly tui: { readonly base: string } | undefined; /** The data directory. */ readonly dataDir: string; /** The owner link of this start (single use, 10 minutes): `http://127.0.0.1:/_chord/owner#`. */ ownerLink(): string; /** The owner token file's path (a device session of the owner, mode 0600). */ readonly ownerTokenFile: string; - /** Builds the dirty plugins and rolls them out: the server, then every worker kind; never by itself. */ + /** Builds the dirty plugins and rolls them out: the server, every worker kind, then the clients; never by itself. */ applyChanges(context: Context): Promise; - /** Stops the worker at `address` and writes its archive (store and files) to `path`. */ - exportAddress(address: Address, path: string, context: Context): Promise; - /** Makes `address` (which must have no data yet) from the archive at `archive`; its next hold starts it. */ - importAddress(archive: string, address: Address, context: Context): Promise; + /** + * Writes the data the worker at `address` created (its store and finished files) as an archive at `path`, stopping + * nothing: a running worker packs its own store at one consistent point; without one the server packs it under the + * worker's lock (no worker starts meanwhile). + */ + exportData(address: Address, path: string, context: Context): Promise; + /** + * Makes the data of `address` (which must have none yet: nothing is overwritten) from the archive at `archive`, + * under the worker's lock; its next hold starts it. + */ + importData(archive: string, address: Address, context: Context): Promise; /** Closes the HTTP component, the workers, the host, the store, the watcher and the data directory's lock. */ close(context: Context): Promise; } @@ -167,12 +193,13 @@ export type KitStart = { * Starts the app's server (§1): resolves once its plugins are loaded and it listens. A failing plugin of the app's * root (or a problem discovery reports there) rejects with the plugins component's `PluginError`, everything closed. */ -export async function startApp

(options: AppOptions

, context: Context): Promise> { +export async function startServer

(options: AppOptions

, context: Context): Promise> { const { layout, cookieSecret, store, lock } = await openServer( resolve(options.dataDir ?? defaultDataDir(options.name)), context, ); const { onError, api, limits = {} } = options; + const sessions = createSessions

(cookieSecret, { maxEphemeral: limits.ephemeralSessions }); const host = createHost

({ address: Server(), principal: options.principal.server, @@ -183,7 +210,7 @@ export async function startApp

(options: AppOptions

, cont host, { listen: options.listen ?? { host: "127.0.0.1", port: 0 }, - links: { path: "/chord" }, + links: { path: "/chord", opened: sessions.opened }, onError: (error) => onError({ kind: "http", error }), }, context, @@ -199,6 +226,7 @@ export async function startApp

(options: AppOptions

, cont ]), ), install: options.install, + publicPath: `${WEB_PREFIX}assets/`, onError: (error) => onError({ kind: "plugins", error }), }, context, @@ -214,7 +242,7 @@ export async function startApp

(options: AppOptions

, cont name: "chord.app.info", facet: appInfoFacet({ dataDir: layout.dataDir, name: options.name, version: api.versions[0] ?? "" }), }, - { name: "chord.app.sessions", facet: sessionsFacet(cookieSecret, { maxEphemeral: limits.ephemeralSessions }) }, + { name: "chord.app.sessions", facet: sessions.facet }, { name: "chord.app.owner", facet: ownerFacet(link) }, { name: "chord.app.pairing", @@ -223,9 +251,16 @@ export async function startApp

(options: AppOptions

, cont ), }, { name: "chord.app.files", facet: filesFacet(join(layout.server, "files"), options.files, limits.uploads) }, + { name: "chord.themes", facet: themesFacet() }, + { name: "chord.themes.choice", facet: themeChoiceFacet() }, ]; - const app = new RunningApp(host, http, workers, plugins, { store, lock }, layout, options); + const surfaces: Surface

[] = []; + const app = new RunningApp(host, http, workers, plugins, surfaces, { store, lock }, layout, options); try { + for (const surface of await surfacesOf(host, options, plugins, context)) { + surfaces.push(surface); + core.push({ name: `chord.${surface.name}`, facet: surface.facet }); + } await plugins.load(host, { kind: "server", core, modules: modulesOf(api, options.kinds.server) }, context); await ownerTokenFile(layout.ownerToken, host, context); app.started(link.renew(), await http.listening(context)); @@ -236,6 +271,30 @@ export async function startApp

(options: AppOptions

, cont return app; } +/** A client surface of the app: its name (`web`, `tui`), its facet, its rollout step, its base bundle's hash. */ +type Surface

= { + readonly name: "web" | "tui"; + readonly facet: Facet

; + step(name: string): RolloutStep; + readonly base: string; +}; + +/** The app's client surfaces, built (their base bundles) in order: the web's, the terminal's. */ +async function surfacesOf

( + host: Host

, + options: AppOptions

, + plugins: Plugins, + context: Context, +): Promise[]> { + const surfaces: Surface

[] = []; + if (options.web !== undefined) + surfaces.push({ name: "web", ...(await webSurface(host, { ...options.web, plugins }, context)) }); + if (options.tui !== undefined) { + surfaces.push({ name: "tui", ...(await tuiSurface(host, { entry: options.tui.entry, plugins }, context)) }); + } + return surfaces; +} + /** How long a second server waits for the data directory's lock before it is refused. */ const SERVER_LOCK_MS = 1000; @@ -251,7 +310,7 @@ async function openServer( const dataDir = await openDataDir(path); const waited = AbortSignal.timeout(SERVER_LOCK_MS); const lock = await processLock(`chord-app:${dataDir}`, withAbortSignal(waited, context)).catch((error: unknown) => { - throw waited.aborted ? new Error(`startApp: another server runs on ${dataDir}`) : error; + throw waited.aborted ? new Error(`startServer: another server runs on ${dataDir}`) : error; }); try { const layout = await createLayout(dataDir); @@ -286,6 +345,8 @@ function workersOf

( context: Context, ): Workers { const facet = kind.facet ?? "worker"; + const serverKey = canonicalJson(options.principal.server as unknown as JsonValue, context); + const isServer = (principal: P): boolean => canonicalJson(principal as unknown as JsonValue, context) === serverKey; return createWorkers( host, // A host kind of the app: any params; its addresses come back from its route as the kit hands them on. @@ -295,7 +356,8 @@ function workersOf

( entry: kind.entry, ...(kind.execArgv === undefined ? {} : { execArgv: kind.execArgv }), ...(kind.env === undefined ? {} : { env: kind.env }), - authorize: kind.authorize, + // The server reaches its workers' kit objects (`HostData`) whatever the app admits. + authorize: (principal, address) => isServer(principal) || kind.authorize(principal, address), principal: options.principal.worker, start: async (address, ctx) => { const kit: KitStart = { @@ -323,10 +385,7 @@ function workerLock(name: string, dataDir: string, address: Address): string { return `${name}:${dataDir}:${address.key}`; } -/** An address `#quiesce` made quiet: its worker's lock, and whether it was failed before. */ -type Quiet = { readonly lock: ProcessLock; readonly failed: boolean }; - -/** How long an export or import waits for a stopped worker's lock before it stops the worker again. */ +/** How long an export or import waits for a worker's lock: past it a worker holds it (it runs or starts). */ const ADDRESS_LOCK_MS = 2000; /** The host of the owner link for the address listened on: loopback for a wildcard, an IPv6 address in brackets. */ @@ -344,6 +403,8 @@ class RunningApp

implements App

{ readonly dataDir: string; readonly ownerTokenFile: string; readonly #component: Plugins; + /** The client surfaces, filled before the server's load. */ + readonly #surfaces: readonly Surface

[]; readonly #store: Store; readonly #lock: ProcessLock; readonly #options: AppOptions

; @@ -355,6 +416,7 @@ class RunningApp

implements App

{ http: HttpComponent

, workers: ReadonlyMap>, plugins: Plugins, + surfaces: readonly Surface

[], server: { readonly store: Store; readonly lock: ProcessLock }, layout: { readonly dataDir: string; readonly plugins: string; readonly ownerToken: string }, options: AppOptions

, @@ -363,6 +425,7 @@ class RunningApp

implements App

{ this.http = http; this.workers = workers; this.#component = plugins; + this.#surfaces = surfaces; this.#store = server.store; this.#lock = server.lock; this.#options = options; @@ -384,72 +447,86 @@ class RunningApp

implements App

{ return this.#link; } + get web(): { readonly base: string } | undefined { + return this.#surfaces.find(({ name }) => name === "web"); + } + + get tui(): { readonly base: string } | undefined { + return this.#surfaces.find(({ name }) => name === "tui"); + } + async applyChanges(context: Context): Promise { const plugins = this.#component; const built = await plugins.build(plugins.state.value.dirty, context); - const steps = [plugins.step("server", this.host), ...[...this.workers].map(([name, each]) => each.step(name))]; + const steps = [ + plugins.step("server", this.host), + ...[...this.workers].map(([name, each]) => each.step(name)), + ...this.#surfaces.map((surface) => surface.step(surface.name)), + ]; return plugins.rollout(steps, built, context); } - async exportAddress(address: Address, path: string, context: Context): Promise { - const quiet = await this.#quiesce(address, context); - try { - const manifest = this.#manifest(address); - await packAddress(workerDirectory(this.dataDir, address), manifest, resolve(path), context); - } finally { - await this.#resume(address, quiet, context); + async exportData(address: Address, path: string, context: Context): Promise { + const workers = this.#workersOf(address); + const manifest = this.#manifest(address); + const target = resolve(path); + // Running: the worker packs its own store. Else the lock, unless a worker starts first: then ask it. + for (let attempt = 1; ; attempt++) { + if (workers.state.value[address.key]?.status === "running") { + const data = await this.host.get(HostData, { address: addressParam(address) }, context); + try { + await data.exportTo({ path: target, manifest: JSON.stringify(manifest) }, context); + } finally { + data.release(); + } + return; + } + const lock = await this.#workerLock(address, context); + if (lock !== undefined) { + try { + await packData(workerDirectory(this.dataDir, address), manifest, target, context); + } finally { + await lock.release(); + } + return; + } + if (attempt === 5) throw new Error(`exportData: the worker at ${address.key} keeps starting; try again`); } } - async importAddress(archive: string, address: Address, context: Context): Promise { + async importData(archive: string, address: Address, context: Context): Promise { + this.#workersOf(address); const { name, limits = {} } = this.#options; const accept = (manifest: ArchiveManifest): void => { // A trust boundary: an archive from anywhere. - if (manifest?.format !== ARCHIVE_FORMAT) - throw new Error(`importAddress: not a format ${ARCHIVE_FORMAT} archive`); - if (manifest.app !== name) throw new Error(`importAddress: an archive of "${manifest.app}", not "${name}"`); + if (manifest?.format !== ARCHIVE_FORMAT) throw new Error(`importData: not a format ${ARCHIVE_FORMAT} archive`); + if (manifest.app !== name) throw new Error(`importData: an archive of "${manifest.app}", not "${name}"`); }; - const quiet = await this.#quiesce(address, context); + const lock = await this.#workerLock(address, context); + // A worker holds it: it runs, so the address has data, and nothing is overwritten. + if (lock === undefined) throw new Error(`importData: ${address.key} has data (its worker runs)`); try { const bounds = { bytes: limits.importBytes, entries: limits.importEntries }; - await unpackAddress(resolve(archive), workerDirectory(this.dataDir, address), accept, bounds, context); + await unpackData(resolve(archive), workerDirectory(this.dataDir, address), accept, bounds, context); } finally { - await this.#resume(address, quiet, context); + await lock.release(); } } - /** - * Takes the process lock of the worker at `address` (the one its worker takes at start) for an export or an import, - * stopping a worker that is not failed first: while it is held, a hold arriving starts a worker that waits for it, - * then opens the finished state. A worker started between the stop and the lock holds the lock first: it is - * stopped again (at most 5 times). A failed address stays failed: it starts nothing (its holds see its failure), - * and `#resume` leaves it so. - */ - async #quiesce(address: Address, context: Context): Promise { - const workers = this.#workersOf(address); - const failed = workers.state.value[address.key]?.status === "failed"; - for (let attempt = 1; ; attempt++) { - if (!failed) await workers.stop(address, context); - const waited = AbortSignal.timeout(ADDRESS_LOCK_MS); - try { - const lock = await processLock( - workerLock(this.#options.name, this.dataDir, address), - withAbortSignal(waited, context), - ); - return { lock, failed }; - } catch (error) { - if (!waited.aborted) throw error; - if (attempt === 5) throw new Error(`app: the worker at ${address.key} keeps starting; try again`); - } + /** The process lock of the worker at `address` (the one its worker takes at start), or `undefined` while held. */ + async #workerLock(address: Address, context: Context): Promise { + const waited = AbortSignal.timeout(ADDRESS_LOCK_MS); + try { + return await processLock( + workerLock(this.#options.name, this.dataDir, address), + withAbortSignal(waited, context), + ); + } catch (error) { + if (waited.aborted) return undefined; + throw error; } } - /** Releases the lock `#quiesce` took, and lets the address start again unless it was failed before. */ - async #resume(address: Address, quiet: Quiet, context: Context): Promise { - await quiet.lock.release(); - if (!quiet.failed) await this.#workersOf(address).retry(address, context); - } - close(context: Context): Promise { this.#closing ??= this.#close(context); return this.#closing; diff --git a/packages/chord/src/next-node/app/archive.ts b/packages/chord/src/next-node/app/archive.ts index 9177bcd76..a0b420f1d 100644 --- a/packages/chord/src/next-node/app/archive.ts +++ b/packages/chord/src/next-node/app/archive.ts @@ -1,14 +1,16 @@ -// Address archives (slice 13 §4, decision "The app kit, point 5" (export and import per address)): one tar of a -// worker's address, `manifest.json` (the app, its API version, the kit's format version), `store.archive` (the store's -// export archive, slice 6a, as JSONL) and `files/` (its files directory). Packing reads a stopped worker's directory; -// unpacking makes a new directory for an address that has none, refusing every path that would leave it. +// Data archives (slice 13 §4, decisions "The app kit, point 5" and "App kit naming and export"): one tar of the data a +// host created, `manifest.json` (the app, its API version, the kit's format version, the address), `store.archive` (the +// store's export archive, slice 6a, as JSONL: one consistent point, the export holding the commit queue while it reads) +// and `files/` (its finished files). A running worker packs its own open store; without one the server packs the +// directory under the worker's lock. Unpacking makes a new directory for an address that has none, refusing every path +// that would leave it. -import { createReadStream, createWriteStream } from "node:fs"; -import { mkdir, mkdtemp, readdir, rename, rm, stat } from "node:fs/promises"; +import { createReadStream, createWriteStream, type Dirent } from "node:fs"; +import { mkdir, mkdtemp, open, readdir, rename, rm, stat } from "node:fs/promises"; import { dirname, join, relative, sep } from "node:path"; import { pipeline } from "node:stream/promises"; import type { Context } from "../../context/index.ts"; -import { decodeJsonl, encodeJsonl, importStore, openStore, sqliteBackend } from "../../next/store/index.ts"; +import { decodeJsonl, encodeJsonl, importStore, openStore, type Store, sqliteBackend } from "../../next/store/index.ts"; import { openNodeSqliteDatabase } from "../../sqlite/node.ts"; import { privateDirectory, privateStore } from "./layout.ts"; import { readTar, type TarEntry, writeTar } from "./tar.ts"; @@ -28,33 +30,44 @@ export type ArchiveManifest = { const STORE = "store.sqlite"; /** - * Packs the host directory `dir` (its worker stopped: nothing writes it) into the archive at `path`: `manifest`, its - * store's export (an empty store's when it has none yet) and its `files/` (uploads under way left out). + * Packs the host directory `dir` of a host that does not run (its worker's lock held: nothing writes it) into the + * archive at `path`: `manifest`, its store's export and its files. */ -export async function packAddress( - dir: string, +export async function packData(dir: string, manifest: ArchiveManifest, path: string, context: Context): Promise { + if (!(await exists(dir))) throw new Error(`exportData: ${manifest.address.key} has no data`); + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, STORE))), context); + try { + await packStore(store, join(dir, "files"), manifest, path, context); + } finally { + await store.close(context); + } +} + +/** + * Packs the open `store` and the files directory `files` into the archive at `path` (mode 0600, staged beside it and + * renamed): `manifest`, the store's export (one consistent point, written while the host runs on) and the finished + * files (uploads under way left out). + */ +export async function packStore( + store: Store, + files: string, manifest: ArchiveManifest, path: string, context: Context, ): Promise { - if (!(await exists(dir))) throw new Error(`exportAddress: ${manifest.address.key} has no data`); const staging = await mkdtemp(join(dirname(path), ".chord-export-")); try { const exported = join(staging, "store.archive"); - const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(join(dir, STORE))), context); - try { - await pipeline(encodeJsonl(store.export(context)), createWriteStream(exported)); - } finally { - await store.close(context); - } + await pipeline(encodeJsonl(store.export(context)), createWriteStream(exported, { mode: 0o600 })); const text = new TextEncoder().encode(JSON.stringify(manifest)); - const entries: TarEntry[] = [ - { path: "manifest.json", size: text.length, data: chunksOf(text) }, - { path: "store.archive", size: (await stat(exported)).size, data: createReadStream(exported) }, - ...(await filesOf(join(dir, "files"))), - ]; + const size = (await stat(exported)).size; + async function* entries(): AsyncGenerator { + yield { path: "manifest.json", size: text.length, data: chunksOf(text) }; + yield { path: "store.archive", size, data: createReadStream(exported) }; + yield* filesOf(files); + } const written = join(staging, "archive.tar"); - await pipeline(writeTar(entries), createWriteStream(written, { mode: 0o600 })); + await pipeline(writeTar(entries()), createWriteStream(written, { mode: 0o600 })); await rename(written, path); } finally { await rm(staging, { recursive: true, force: true }); @@ -66,30 +79,52 @@ async function* chunksOf(bytes: Uint8Array): AsyncGenerator { yield bytes; } -/** The files under `root` as entries `files/` (hidden ones, uploads under way, left out). */ -async function filesOf(root: string): Promise { - const entries: TarEntry[] = []; +/** + * The files under `root` as entries `files/` (hidden ones, uploads under way, left out), each opened only + * when the archive comes to it (one descriptor at a time) and its size taken from the open file: a file deleted while a + * running worker is exported is skipped, or read whole from its open handle (finished files never change; Opus L7). + */ +async function* filesOf(root: string): AsyncGenerator { + for (const path of await pathsUnder(root)) { + const handle = await open(path, "r").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); + if (handle === undefined) continue; + const info = await handle.stat(); + if (!info.isFile()) { + await handle.close(); + continue; + } + const inside = relative(root, path).split(sep).join("/"); + yield { + path: `files/${inside}`, + size: info.size, + data: handle.createReadStream({ start: 0, end: Math.max(0, info.size - 1) }), + }; + } +} + +/** The paths of the files under `root`, sorted, hidden ones left out (a directory gone meanwhile: none of it). */ +async function pathsUnder(root: string): Promise { + const paths: string[] = []; const walk = async (dir: string): Promise => { - let names: string[]; + let entries: Dirent[]; try { - names = (await readdir(dir)).sort(); + entries = (await readdir(dir, { withFileTypes: true })).sort((a, b) => (a.name < b.name ? -1 : 1)); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error; } - for (const name of names) { - if (name.startsWith(".")) continue; - const path = join(dir, name); - const info = await stat(path); - if (info.isDirectory()) await walk(path); - else if (info.isFile()) { - const inside = relative(root, path).split(sep).join("/"); - entries.push({ path: `files/${inside}`, size: info.size, data: createReadStream(path) }); - } + for (const entry of entries) { + if (entry.name.startsWith(".")) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) await walk(path); + else if (entry.isFile()) paths.push(path); } }; await walk(root); - return entries; + return paths; } /** What an import takes at most: unpacked bytes (4 GiB by default) and entries (100,000 by default). */ @@ -100,14 +135,14 @@ export type ImportLimits = { readonly bytes?: number; readonly entries?: number * `accept` (a throw refuses it), the store made from `store.archive`, `files/` restored, within `limits`. Staged beside * `dir` (mode 0700, files 0600) and renamed into place at the end, so a refused or broken archive leaves nothing. */ -export async function unpackAddress( +export async function unpackData( archive: string, dir: string, accept: (manifest: ArchiveManifest) => void, limits: ImportLimits, context: Context, ): Promise { - if (await exists(dir)) throw new Error(`importAddress: ${dir} exists already (an address is imported once)`); + if (await exists(dir)) throw new Error(`importData: ${dir} exists already (an address is imported once)`); await privateDirectory(dirname(dir)); const staging = await mkdtemp(join(dirname(dir), ".chord-import-")); try { @@ -144,12 +179,12 @@ async function unpackEntries( let entries = 0; for await (const entry of readTar(createReadStream(archive))) { context.abortSignal?.throwIfAborted(); - if (++entries > maxEntries) throw new Error(`importAddress: the archive has more than ${maxEntries} entries`); + if (++entries > maxEntries) throw new Error(`importData: the archive has more than ${maxEntries} entries`); if (entry.kind === "directory") continue; bytes += entry.size; - if (bytes > maxBytes) throw new Error(`importAddress: the archive unpacks to more than ${maxBytes} bytes`); + if (bytes > maxBytes) throw new Error(`importData: the archive unpacks to more than ${maxBytes} bytes`); if (manifest === undefined && entry.path !== "manifest.json") { - throw new Error("importAddress: the archive does not start with manifest.json"); + throw new Error("importData: the archive does not start with manifest.json"); } if (entry.path === "manifest.json") { // A trust boundary: the archive's own description, checked by `accept`. @@ -164,11 +199,11 @@ async function unpackEntries( await mkdir(dirname(target), { recursive: true, mode: 0o700 }); await pipeline(entry.data, createWriteStream(target, { mode: 0o600 })); } else { - throw new Error(`importAddress: the archive holds ${entry.path}, which an address archive does not`); + throw new Error(`importData: the archive holds ${entry.path}, which an address archive does not`); } } if (manifest === undefined || storeArchive === undefined) { - throw new Error("importAddress: the archive has no manifest.json or store.archive"); + throw new Error("importData: the archive has no manifest.json or store.archive"); } return storeArchive; } @@ -179,7 +214,7 @@ async function collect(data: AsyncIterable, limit: number): Promise< let size = 0; for await (const chunk of data) { size += chunk.length; - if (size > limit) throw new Error("importAddress: manifest.json is too large"); + if (size > limit) throw new Error("importData: manifest.json is too large"); chunks.push(chunk); } return Buffer.concat(chunks); diff --git a/packages/chord/src/next-node/app/data.ts b/packages/chord/src/next-node/app/data.ts new file mode 100644 index 000000000..df18e6dce --- /dev/null +++ b/packages/chord/src/next-node/app/data.ts @@ -0,0 +1,52 @@ +// A host's data on its worker (decision "App kit naming and export"): the kit's object a running worker serves, through +// which the server exports the worker's data without stopping it. The worker packs its own open store (its export holds +// the commit queue while it reads: one consistent point) and its finished files into the archive at the path the server +// names. Only the server may ask (its principal); the worker's plugins never see it. + +import { BACKGROUND_CONTEXT } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { type Address, method, obj, remote, str } from "../../next/contracts/index.ts"; +import { ChordError, type Principal } from "../../next/data/index.ts"; +import { canonicalJson } from "../../next/data/principal.ts"; +import { defineFacet, type Facet } from "../../next/host/index.ts"; +import type { Store } from "../../next/store/index.ts"; +import { type ArchiveManifest, packStore } from "./archive.ts"; + +/** The data of the worker at `address` (its JSON, as `addressParam` writes it): `exportTo` packs it at `path`. */ +export const HostData = remote("chord.app.data", { + version: 1, + params: { address: str() }, + // The address a worker has: the server names it (`addressParam`); only its kind and params place the object. + hostAddress: ({ address }) => JSON.parse(address) as Address, + variants: { main: { methods: { exportTo: method(obj({ path: str(), manifest: str() })) } } }, +}); + +/** `address` as `HostData`'s param. */ +export function addressParam(address: Address): string { + return JSON.stringify({ kind: address.kind, params: address.params, key: address.key }); +} + +/** + * The kit's worker facet serving `HostData` for the worker's open `store` and its files directory `files`, to `server` + * (the server's principal) only. + */ +export function hostDataFacet

(store: Store, files: string, server: P): Facet

{ + const serverKey = canonicalJson(server as JsonValue, BACKGROUND_CONTEXT); + return defineFacet

((host) => + host.serve(HostData, (_params, principal) => { + if (canonicalJson(principal as JsonValue, BACKGROUND_CONTEXT) !== serverKey) { + throw new ChordError({ code: "forbidden", message: "chord.app.data: the server's only" }); + } + return { + variant: "main", + members: { + exportTo: async ({ path, manifest }, ctx) => { + // The server's own manifest for this address. + await packStore(store, files, JSON.parse(manifest) as ArchiveManifest, path, ctx); + return null; + }, + }, + }; + }), + ); +} diff --git a/packages/chord/src/next-node/app/ephemeral-sessions.ts b/packages/chord/src/next-node/app/ephemeral-sessions.ts new file mode 100644 index 000000000..a64f6404a --- /dev/null +++ b/packages/chord/src/next-node/app/ephemeral-sessions.ts @@ -0,0 +1,80 @@ +// Ephemeral sessions (slice 13 §3, carry-over to slice 11): a pick-a-name login's sessions, in memory only, at most +// `max` (past it the oldest ends), and each ending with its client's connection: once the last Chord link +// authenticated with it closed and none opened for a grace period (a page reload, a reconnect after a drop), or when +// none opened within that period of the login. + +import type { Principal } from "../../next/data/index.ts"; + +/** How long an ephemeral session lives without a link: a minute, past a reload and the reconnect loop's pauses. */ +export const EPHEMERAL_GRACE_MS = 60_000; + +/** One ephemeral session: its secret's hash, its principal, its open links, its end's timer while it has none. */ +type Session = { + readonly hash: string; + readonly principal: Principal; + links: number; + timer: ReturnType | undefined; +}; + +/** The server's ephemeral sessions. */ +export class EphemeralSessions { + readonly #max: number; + readonly #graceMs: number; + /** Insertion order is age order: the first entry is the oldest. */ + readonly #sessions = new Map(); + + constructor(max: number, graceMs = EPHEMERAL_GRACE_MS) { + this.#max = max; + this.#graceMs = graceMs; + } + + /** A new session `id` (its secret's hash `hash`) for `principal`; the oldest ends past the cap. */ + issue(id: string, hash: string, principal: Principal): void { + const session: Session = { hash, principal, links: 0, timer: undefined }; + this.#sessions.set(id, session); + this.#waitForLink(id, session); + for (const [oldest] of this.#sessions) { + if (this.#sessions.size <= this.#max) break; + this.revoke(oldest); + } + } + + /** Session `id`'s hash and principal, while it lives. */ + get(id: string): { readonly hash: string; readonly principal: Principal } | undefined { + return this.#sessions.get(id); + } + + /** Ends session `id`; whether it was one. */ + revoke(id: string): boolean { + const session = this.#sessions.get(id); + if (session === undefined) return false; + clearTimeout(session.timer); + this.#sessions.delete(id); + return true; + } + + /** A link authenticated with session `id` opened: what to call when it closes; `undefined` for no such session. */ + linkOpened(id: string): (() => void) | undefined { + const session = this.#sessions.get(id); + if (session === undefined) return undefined; + session.links++; + clearTimeout(session.timer); + session.timer = undefined; + let closed = false; + return () => { + if (closed) return; + closed = true; + session.links--; + if (session.links === 0 && this.#sessions.get(id) === session) this.#waitForLink(id, session); + }; + } + + /** Ends `session` unless a link opens within the grace period. */ + #waitForLink(id: string, session: Session): void { + session.timer = setTimeout(() => { + if (session.links === 0) this.revoke(id); + }, this.#graceMs); + // The server's process does not wait for it. + session.timer.unref?.(); + } +} diff --git a/packages/chord/src/next-node/app/index.ts b/packages/chord/src/next-node/app/index.ts index aff1f6f71..2eb6e1b4b 100644 --- a/packages/chord/src/next-node/app/index.ts +++ b/packages/chord/src/next-node/app/index.ts @@ -1,4 +1,4 @@ -// Chord v2's app kit (`@earendil-works/chord/next/app` once v2 has its entry point; slice 13): `startApp` on the server +// Chord v2's app kit (`@earendil-works/chord/next/app` once v2 has its entry point; slice 13): `startServer` on the server // and `startWorker` in a worker's entry, the start-data objects, sessions and the owner's sign-in, device pairing, the // files of each host, export and import per address, and the optional plugins (users, invites, ephemeral login). @@ -11,7 +11,7 @@ export { type AppPlugins, type AppWorkerKind, Server, - startApp, + startServer, } from "./app.ts"; export { ARCHIVE_FORMAT, type ArchiveManifest } from "./archive.ts"; export { type EphemeralUser, ephemeralLoginFacet } from "./ephemeral.ts"; diff --git a/packages/chord/src/next-node/app/layout.ts b/packages/chord/src/next-node/app/layout.ts index 063edd825..ad3b379fa 100644 --- a/packages/chord/src/next-node/app/layout.ts +++ b/packages/chord/src/next-node/app/layout.ts @@ -38,11 +38,11 @@ export async function openDataDir(path: string): Promise { const home = await realpath(homedir()); if (dataDir === home || dataDir === parse(dataDir).root) { const what = dataDir === home ? "the home directory" : "a file system's root"; - throw new Error(`startApp: the data directory ${dataDir} is ${what}; name a directory of the app's own`); + throw new Error(`startServer: the data directory ${dataDir} is ${what}; name a directory of the app's own`); } const { uid } = await stat(dataDir); if (process.getuid !== undefined && uid !== process.getuid()) { - throw new Error(`startApp: the data directory ${dataDir} belongs to another user (uid ${uid})`); + throw new Error(`startServer: the data directory ${dataDir} belongs to another user (uid ${uid})`); } return dataDir; } diff --git a/packages/chord/src/next-node/app/sessions.ts b/packages/chord/src/next-node/app/sessions.ts index f42a4e5a1..88ce03bba 100644 --- a/packages/chord/src/next-node/app/sessions.ts +++ b/packages/chord/src/next-node/app/sessions.ts @@ -3,8 +3,9 @@ // the secret's hash is kept. A browser carries its token in the kit's signed session cookie (`HttpOnly`, // `SameSite=Strict`), a command-line client in `Authorization: Bearer `. Browser and device sessions persist in // the server's store and expire after 400 days unused; ephemeral sessions (a pick-a-name login, a public route) live -// in memory only, at most `maxEphemeral` of them (past it the oldest ends), so a public login costs the same whatever -// came before, and none outlives the server's run. +// in memory only (`EphemeralSessions`), at most `maxEphemeral` of them (past it the oldest ends), so a public login +// costs the same whatever came before; each ends with its client's connection (its last link closed, a grace period +// passed: the links endpoint's `opened` hook counts them), and none outlives the server's run. import type { Context } from "../../context/index.ts"; import type { JsonValue } from "../../json.ts"; @@ -22,6 +23,7 @@ import { readCookie, readSignedCookie, } from "../http/index.ts"; +import { EphemeralSessions } from "./ephemeral-sessions.ts"; import { hashToken, matchesHash, newToken } from "./secrets.ts"; /** What a session is for: a browser (its cookie), a device (a command-line client's token), an ephemeral login. */ @@ -96,9 +98,6 @@ const SessionsDocument = defineState("chord.app.sessions", { checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 50, }); -/** An ephemeral session, in memory: its secret's hash and its principal. */ -type EphemeralSession = { readonly hash: string; readonly principal: Principal }; - /** How often a session's `lastUsed` is written at most: once a minute (not one commit per request). */ const TOUCH_MS = 60_000; @@ -108,17 +107,33 @@ const SESSION_IDLE_MS = 400 * 24 * 60 * 60_000; /** The sessions' limits: how many ephemeral sessions live at once (1,000 by default; past it the oldest ends). */ export type SessionLimits = { readonly maxEphemeral?: number }; +/** The kit's sessions: the facet for the server's core facets, and the links endpoint's `opened` hook. */ +export type KitSessions

= { + readonly facet: Facet

; + /** A link opened with `request`'s credentials: an ephemeral session's links are counted (it ends with its last). */ + readonly opened: (request: HttpRequest) => (() => void) | undefined; +}; + /** - * The kit's facet serving `Sessions` (a host-lifetime object) and contributing its authenticator: the bearer token - * first, else the session cookie signed with `cookieSecret`. + * The kit's sessions: its facet serving `Sessions` (a host-lifetime object) and contributing its authenticator (the + * bearer token first, else the session cookie signed with `cookieSecret`), and the links hook ending an ephemeral + * session with its client's connection. */ -export function sessionsFacet

(cookieSecret: string, limits: SessionLimits = {}): Facet

{ - const maxEphemeral = limits.maxEphemeral ?? 1000; +export function createSessions

(cookieSecret: string, limits: SessionLimits = {}): KitSessions

{ + const ephemeral = new EphemeralSessions(limits.maxEphemeral ?? 1000); + const opened = (request: HttpRequest): (() => void) | undefined => { + const token = bearerToken(request) ?? readCookie(request, SESSION_COOKIE, cookieSecret); + const dot = token?.indexOf(".") ?? -1; + return dot > 0 ? ephemeral.linkOpened(token!.slice(0, dot)) : undefined; + }; + return { facet: sessionsFacet

(cookieSecret, ephemeral), opened }; +} + +/** The facet of `createSessions`. */ +function sessionsFacet

(cookieSecret: string, ephemeral: EphemeralSessions): Facet

{ return defineFacet

((host) => host.serve(Sessions, (_params, self) => { const document = self.state(SessionsDocument); - // Insertion order is age order: the first entry is the oldest. - const ephemeral = new Map(); const change = (fn: (sessions: Record) => void, context: Context) => // Not awaited by authentication: the request goes on; a write that fails is made again by a later request. void document.change((draft) => fn(draft.sessions), context).catch(() => {}); @@ -172,11 +187,7 @@ export function sessionsFacet

(cookieSecret: string, limits: const secret = newToken(32); if (kind === "ephemeral") { // Principals are the app's: `issue`'s callers pass one of its type. - ephemeral.set(id, { hash: hashToken(secret), principal: principal as Principal }); - for (const [oldest] of ephemeral) { - if (ephemeral.size <= maxEphemeral) break; - ephemeral.delete(oldest); - } + ephemeral.issue(id, hashToken(secret), principal as Principal); } else { const now = Date.now(); const stored = JSON.stringify(principal); @@ -199,7 +210,7 @@ export function sessionsFacet

(cookieSecret: string, limits: }, verify: async (token, ctx) => (authenticate(token, ctx) as JsonValue | undefined) ?? null, revoke: async (id, ctx) => { - if (ephemeral.delete(id)) return null; + if (ephemeral.revoke(id)) return null; await document.change((draft) => { delete draft.sessions[id]; }, ctx); diff --git a/packages/chord/src/next-node/app/worker.ts b/packages/chord/src/next-node/app/worker.ts index edb398434..d9addad45 100644 --- a/packages/chord/src/next-node/app/worker.ts +++ b/packages/chord/src/next-node/app/worker.ts @@ -12,11 +12,12 @@ import { openNodeSqliteDatabase } from "../../sqlite/node.ts"; import type { Watchable } from "../../watch.ts"; import { workerMain } from "../workers/index.ts"; import type { AppApi, AppKind, KitStart } from "./app.ts"; +import { hostDataFacet } from "./data.ts"; import { filesFacet } from "./files.ts"; import { workerInfoFacet } from "./info.ts"; import { privateDirectory, privateStore } from "./layout.ts"; -/** What `startWorker` takes: the app's API and kinds as the server's `startApp` has them, this process's modules. */ +/** What `startWorker` takes: the app's API and kinds as the server's `startServer` has them, this process's modules. */ export type WorkerOptions = { readonly api: AppApi; readonly kinds: Readonly>; @@ -32,7 +33,7 @@ export function startWorker

(options: WorkerOptions): void { const packages = Object.assign({}, ...Object.values(options.kinds).map((kind) => kind.packages ?? {})); workerMain( async ({ address, start, parent, plugins, onClose }, context) => { - // The server's `startApp` sends `{ kit, app }`. + // The server's `startServer` sends `{ kit, app }`. const { kit, app } = start as { readonly kit: KitStart; readonly app: JsonValue }; await privateDirectory(kit.dir); await privateStore(join(kit.dir, "store.sqlite")); @@ -58,6 +59,10 @@ export function startWorker

(options: WorkerOptions): void { const core = [ { name: "chord.app.worker-info", facet: workerInfoFacet

(info) }, { name: "chord.app.files", facet: filesFacet

(join(kit.dir, "files"), kit.files, kit.uploads) }, + { + name: "chord.app.data", + facet: hostDataFacet

(store, join(kit.dir, "files"), kit.server as unknown as P), + }, ]; await plugins.load(host, { kind: kit.facet, core }, context); return options.busy === undefined ? {} : { busy: options.busy }; diff --git a/packages/chord/src/next-node/http/README.md b/packages/chord/src/next-node/http/README.md index 8d482c464..fbb51a58e 100644 --- a/packages/chord/src/next-node/http/README.md +++ b/packages/chord/src/next-node/http/README.md @@ -15,7 +15,8 @@ Exported as `@earendil-works/chord/next/http` once v2 has its entry point. // the app's settings are arguments, fixed for the component's life (changing one is a restart) const http = createHttp(server, { listen: { host: "127.0.0.1", port: 8788 }, // Chord's own listener; port 0 picks one; none: no listener - links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }) }, // or a value + links: { path: "/chord", options: (request, principal) => ({ limits, maxPayload }), // or a value + opened: (request, principal) => () => linkClosed() }, // called per accepted link; its return at its close closeTimeout: 30_000, onError: (error) => log(error), // HttpError }, ctx); @@ -50,6 +51,7 @@ self.contribute(Http, { framers: ["https://shell.example.com"], // who may frame the app's pages relays: [{ name: "radius", verify: (request) => verified(request) }], // { origin, framer? } | undefined csp: { "img-src": ["https://cdn.example.com"] }, + scripts: ["/_chord/web/"], // script-src: these paths of the origin (and the nonce), not all of it }); ``` @@ -136,7 +138,8 @@ await http.close(ctx); // before host.close - **Default headers** on every answer, the component's own 403, 404 and 503 included: `Cache-Control: no-store`, `X-Content-Type-Options: nosniff`, `Referrer-Policy: no-referrer`, `X-Frame-Options: DENY` (without framers), a CSP (`default-src 'none'`, scripts `'self'` and the response's own nonce, `connect-src` self and its WebSocket, `base-uri - 'none'`, `form-action 'self'`, `frame-ancestors` the framers or `'none'`, plus contributed sources). A response's own + 'none'`, `form-action 'self'`, `frame-ancestors` the framers or `'none'`, plus contributed sources; with contributed + `scripts` paths, `script-src` is those paths of the request's origin and the nonce instead of `'self'`). A response's own headers replace same-named ones (a list sends one line per value: several `Set-Cookie`); a header with a CR, LF or NUL, or a value that is no JSON, is a 500 reported `response_failed`. - **Sockets:** after admission, the origin checks and the authenticators (no principal and not public: 401 before diff --git a/packages/chord/src/next-node/http/headers.ts b/packages/chord/src/next-node/http/headers.ts index 4358e8140..cd79f088a 100644 --- a/packages/chord/src/next-node/http/headers.ts +++ b/packages/chord/src/next-node/http/headers.ts @@ -119,7 +119,7 @@ function csp(admitted: Admitted | undefined, router: Router | undefined): readon if (admitted?.framer !== undefined) framers.push(admitted.framer); const directives = new Map([ ["default-src", ["'none'"]], - ["script-src", ["'self'", NONCE]], + ["script-src", [...scriptSources(admitted, router), NONCE]], ["connect-src", ["'self'", ...(admitted === undefined ? [] : [admitted.origin.replace(/^http/, "ws")])]], ["base-uri", ["'none'"]], ["form-action", ["'self'"]], @@ -138,5 +138,12 @@ function csp(admitted: Admitted | undefined, router: Router | undefined): readon return [`${text.slice(0, at)}'nonce-`, `'${text.slice(at + NONCE.length)}`]; } +/** Where scripts may come from: the contributed script paths of the request's origin, else the whole origin. */ +function scriptSources(admitted: Admitted | undefined, router: Router | undefined): string[] { + const paths = router?.scripts ?? []; + if (admitted === undefined || paths.length === 0) return ["'self'"]; + return paths.map((path) => `${admitted.origin}${path}`); +} + /** Where the nonce goes in the CSP's text while it is made. */ const NONCE = "\0nonce\0"; diff --git a/packages/chord/src/next-node/http/http.ts b/packages/chord/src/next-node/http/http.ts index 4ebb25abe..e21271529 100644 --- a/packages/chord/src/next-node/http/http.ts +++ b/packages/chord/src/next-node/http/http.ts @@ -46,6 +46,11 @@ export type HttpOptions = { export type LinksOptions = { readonly path: string; readonly options?: LinkOptions | ((request: HttpRequest, principal: Principal) => LinkOptions); + /** + * Called when a link is accepted, with its request and principal; what it returns is called when that link closes + * (the app tracking its clients' links: the kit ends an ephemeral session with its last link). + */ + readonly opened?: (request: HttpRequest, principal: Principal) => (() => void) | undefined; }; /** What the component reports. */ @@ -59,7 +64,7 @@ export type HttpError = | { readonly kind: "hook_failed"; readonly hook: HookName; readonly plugin?: string; readonly error: unknown }; /** The hooks the component calls. */ -export type HookName = "authenticate" | "relay" | "links.options"; +export type HookName = "authenticate" | "relay" | "links.options" | "links.opened"; /** The running component. */ export interface HttpComponent

{ @@ -437,7 +442,15 @@ class Component

implements HttpComponent

, Handler { return refuse(stream, 403); } const admitted = { allowed, relay, principal, ...options }; - return this.#upgrades.link(message, stream, head, admitted, this.#host, this.#context); + const opened = (): (() => void) | undefined => { + try { + return links.opened?.(webRequest, principal); + } catch (error) { + this.#hookFailed("links.opened", undefined, error); + return undefined; + } + }; + return this.#upgrades.link(message, stream, head, admitted, this.#host, this.#context, opened); } const match = router.socket(webRequest.path); if (match === "bad_path") return refuse(stream, 400); diff --git a/packages/chord/src/next-node/http/router.ts b/packages/chord/src/next-node/http/router.ts index 1aa1debac..846fb3b80 100644 --- a/packages/chord/src/next-node/http/router.ts +++ b/packages/chord/src/next-node/http/router.ts @@ -80,6 +80,12 @@ export type HttpContribution = { readonly relays?: readonly HttpRelay[]; /** Extra CSP sources by directive (`{ "img-src": ["https://cdn.example.com"] }`). */ readonly csp?: Readonly>; + /** + * The paths scripts load from (`/_chord/web/`): with any, `script-src` is these paths of the request's origin and the + * response's nonce instead of the whole origin, so no other same-origin response (an upload served as JavaScript) + * runs as a script. + */ + readonly scripts?: readonly string[]; }; /** The router a request uses: as it was when the request started. */ @@ -96,6 +102,8 @@ export type Router = { readonly relays: readonly HttpRelay[]; /** Extra CSP sources by directive. */ readonly csp: ReadonlyMap; + /** The paths scripts load from; none: the whole origin. */ + readonly scripts: readonly string[]; /** The socket handlers it serves: a socket handed out by one no longer here is closed. */ readonly socketHandlers: ReadonlySet; }; @@ -147,11 +155,13 @@ export function composeRouter(table: RegistryTable, { report } const framers = new Set(); const relays: HttpRelay[] = []; const csp = new Map(); + const scripts: string[] = []; const socketHandlers = new Set(); for (const { plugin, value } of table) { for (const host of value.hosts ?? []) hosts.add(host.toLowerCase()); for (const framer of value.framers ?? []) framers.add(framer); relays.push(...(value.relays ?? [])); + scripts.push(...(value.scripts ?? [])); for (const [directive, sources] of Object.entries(value.csp ?? {})) { csp.set(directive, [...(csp.get(directive) ?? []), ...sources]); } @@ -215,6 +225,7 @@ export function composeRouter(table: RegistryTable, { report } framers, relays, csp, + scripts, socketHandlers, }; } diff --git a/packages/chord/src/next-node/http/upgrades.ts b/packages/chord/src/next-node/http/upgrades.ts index 820ba83fa..bc10751c2 100644 --- a/packages/chord/src/next-node/http/upgrades.ts +++ b/packages/chord/src/next-node/http/upgrades.ts @@ -48,7 +48,8 @@ export class Upgrades { /** * Upgrades `message` to the links endpoint: `host.accept` over the socket (its output capped) as the admitted - * principal, with `options`' limits; `context` is the parent of the work on the link. + * principal, with `options`' limits; `context` is the parent of the work on the link. `opened` runs once the link + * is accepted; what it returns runs when its socket closes. */ link

( message: IncomingMessage, @@ -57,6 +58,7 @@ export class Upgrades { admission: Admission

& LinkOptions, host: Host

, context: Context, + opened: () => (() => void) | undefined, ): void { const { principal, limits } = admission; upgrade(message, stream, head, admission.maxPayload ?? DEFAULT_LINK_PAYLOAD, (socket) => { @@ -69,6 +71,8 @@ export class Upgrades { seal: () => link.close(), }); host.accept(link, { principal, ...(limits === undefined ? {} : { limits }) }, context); + const closed = opened(); + if (closed !== undefined) socket.once("close", closed); }); } diff --git a/packages/chord/src/next-node/plugins/README.md b/packages/chord/src/next-node/plugins/README.md index 1a6965394..a79421bc1 100644 --- a/packages/chord/src/next-node/plugins/README.md +++ b/packages/chord/src/next-node/plugins/README.md @@ -60,7 +60,9 @@ await plugins.rollout([plugins.step("server", server), workers.step("workers")], it; an ordinary library names files as it likes) is refused naming the chain. A plugin's root may lie under a `node_modules` (an app installed from npm with its plugins): only the path below a plugin's directory counts. A copied file imports host packages and its own plugin's files only, no builtins. A single-file plugin imports host packages only. External: host packages, `@earendil-works/chord/next/plugins`, `chord.external` - packages and Node builtins (Node builds; a browser build refuses both). Browser builds collect CSS into one stylesheet (`css`) and + packages and Node builtins (Node builds; a browser build refuses both). Browser builds collect CSS into one stylesheet (`css`; a `*.module.css` file scoped to its + plugin and file: its classes renamed `chord___`, its import esbuild's mapping, `composes` and + escaped names included, `css-modules.ts`; a stylesheet's `url()` is an asset, an `@import`ed file plain CSS) and assets (`import url from "./icon.svg?url"`, `.wasm`) into content-hashed files (`assets`) under `publicPath`. Refused, with the file and line: `import.meta`, `__dirname`, `__filename` (each use, found through the script's source map), a literal dynamic `import("x")` and top-level `await` (pointing to `pluginFile`; a computed `import(x)` @@ -166,5 +168,6 @@ dependency pinned to a commit, per-kind contract files, a required root), `51-si per-kind files, roots, and input capture), `evaluate.ts` (evaluation, the `require` a script gets, `pluginFile`), `stack.ts` (the source maps of evaluated scripts while their facets live, `mapStack`), `install.ts` (dependencies, git folder packages, plugins from git and npm), `watcher.ts` (the watcher and its marks, a JSON watchable), -`runtime.ts` (per-host loaded facets, complete-set loads and reloads, required roots, no esbuild), `plugins.ts` (the +`css-modules.ts` (CSS modules scoped per plugin and file), `runtime.ts` (evaluated scripts as complete sets for the +portable `LoadedFacets` of `next/surfaces`, required roots, no esbuild), `plugins.ts` (the component: kinds, current sets, `build`, `load`, `reload`, `step`, `rollout`, installs), `testing.ts` (`testPlugin`). diff --git a/packages/chord/src/next-node/plugins/build.ts b/packages/chord/src/next-node/plugins/build.ts index dbcb319e2..adffb2e8a 100644 --- a/packages/chord/src/next-node/plugins/build.ts +++ b/packages/chord/src/next-node/plugins/build.ts @@ -10,8 +10,16 @@ import { createHash } from "node:crypto"; import { readFile, realpath } from "node:fs/promises"; import { isBuiltin } from "node:module"; import { basename, dirname, extname, isAbsolute, join, relative, sep } from "node:path"; -import { type BuildFailure, build, type Plugin as EsbuildPlugin, type Loader, type Message } from "esbuild"; +import { + type BuildFailure, + build, + type Plugin as EsbuildPlugin, + type Loader, + type Message, + type OnLoadResult, +} from "esbuild"; import type { Context } from "../../context/index.ts"; +import { SCOPED_CSS, scopeCssModule } from "./css-modules.ts"; import { discoverPackage, type Plugin } from "./discover.ts"; import { PLUGINS_SPECIFIER, packageName } from "./evaluate.ts"; import { inlineSourceMap } from "./stack.ts"; @@ -255,6 +263,8 @@ class ImportRules { readonly #hostPackages: ReadonlySet; readonly #inputs = new Map(); readonly #copied = new Set(); + /** The scoped stylesheet of every CSS module read, by its file. */ + readonly #scoped = new Map(); /** The first importer of every file resolved, by file. */ readonly #importers = new Map(); /** The API versions imported, each with its first importer. */ @@ -278,6 +288,9 @@ class ImportRules { name: "chord-plugin-imports", setup: (builder) => { builder.onResolve({ filter: /.*/ }, async (args) => { + if (args.path.startsWith(`${SCOPED_CSS}:`)) { + return { path: args.path.slice(SCOPED_CSS.length + 1), namespace: SCOPED_CSS }; + } // The plugin's files are resolved here, the entry too: esbuild then gives them no Node module type // (`"type": "module"`), whose interop would read a whole host package for a default import instead // of honouring the `__esModule` an ESM namespace is wrapped with (§3). @@ -326,18 +339,34 @@ class ImportRules { return { path: found.path, pluginData: resolved.asset ? ASSET : undefined }; }); builder.onLoad({ filter: /.*/ }, async (args) => { + if (args.namespace === SCOPED_CSS) { + // Its `url()` and `@import` resolve relative to the module file. + return { contents: this.#scoped.get(args.path) ?? "", loader: "css", resolveDir: dirname(args.path) }; + } if (args.namespace !== "file") return undefined; const asset = args.pluginData === ASSET; const loader = asset ? "file" : LOADERS[extname(args.path)]; if (loader === undefined || this.#place(args.path).kind === "package") return undefined; const contents = await readFile(args.path); this.#inputs.set(args.path, createHash("sha256").update(contents).digest("hex")); + if (!asset && args.path.endsWith(".module.css")) return this.#cssModule(args.path, contents); return { contents, loader, resolveDir: dirname(args.path) }; }); }, }; } + /** + * The CSS module `file` with `contents` as the facet imports it: a module whose default export is the class mapping, + * importing the scoped stylesheet (`scopeCssModule`), which the build collects with the facet's other CSS. + */ + async #cssModule(file: string, contents: Uint8Array): Promise { + const scoped = await scopeCssModule(this.#own.name, this.#own.dir, file, new TextDecoder().decode(contents)); + this.#scoped.set(file, scoped.css); + const module = `import ${JSON.stringify(`${SCOPED_CSS}:${file}`)};export default ${JSON.stringify(scoped.classes)};`; + return { contents: module, loader: "js", resolveDir: dirname(file) }; + } + /** Every plugin file read, sorted by path. */ inputs(): BuildInput[] { return [...this.#inputs].map(([file, hash]) => ({ file, hash })).sort((a, b) => (a.file < b.file ? -1 : 1)); @@ -412,7 +441,9 @@ class ImportRules { return { kind: "error", text: `"${specifier}": assets by URL are for browser builds (${PLUGIN_FILE_HINT})` }; } if (from.kind === "package") return { kind: "package" }; - if (specifier.startsWith(".") || isAbsolute(specifier)) return { kind: "path", path: specifier, asset: false }; + // A stylesheet's `url()` names an asset of the plugin (served under `publicPath`). + const asset = kind === "url-token" && browser; + if (specifier.startsWith(".") || isAbsolute(specifier)) return { kind: "path", path: specifier, asset }; const name = packageName(specifier); if (this.#own.external.includes(name)) { if (!browser) return { kind: "external" }; diff --git a/packages/chord/src/next-node/plugins/css-modules.ts b/packages/chord/src/next-node/plugins/css-modules.ts new file mode 100644 index 000000000..a5670cdaa --- /dev/null +++ b/packages/chord/src/next-node/plugins/css-modules.ts @@ -0,0 +1,65 @@ +// CSS modules in plugin builds (slice 11 §4b, decision "Looks and themes"; fix round 1, Opus L1): a plugin's +// `*.module.css` has its class names made unique to the file, the import giving the mapping, so twenty plugins' +// `.card` never collide. esbuild's `local-css` loader names classes uniquely within one build only (and a minified +// build names them `.a`, `.b`), while each facet is a build of its own; so each CSS module is scoped by a small build of +// its own whose names start with a prefix of the plugin and the file, and the facet's build gets the scoped stylesheet +// and the mapping. The mapping is esbuild's own (`composes` and escaped names as esbuild gives them); `url()` and +// `@import` stay as written and resolve in the facet's build, relative to the module (an `@import`ed file is plain CSS, +// its classes not scoped). + +import { createHash } from "node:crypto"; +import { basename, relative } from "node:path"; +import { Script } from "node:vm"; +import { build } from "esbuild"; + +/** A CSS module scoped: its stylesheet with unique class names, and the mapping its import gives. */ +export type ScopedCss = { readonly css: string; readonly classes: Readonly> }; + +/** The namespace of a scoped stylesheet the facet's build imports. */ +export const SCOPED_CSS = "chord-css-module"; + +/** + * Scopes the CSS module `file` of the plugin `plugin` (its directory `dir`) with `contents`: every class renamed + * `chord___`. Throws esbuild's failure (a syntax error) for the build to report. + */ +export async function scopeCssModule(plugin: string, dir: string, file: string, contents: string): Promise { + const scope = createHash("sha256") + .update(`${plugin}\0${relative(dir, file)}`) + .digest("hex") + .slice(0, 8); + // esbuild names a local class `_`: the file name carries the prefix. + const name = `chord_${scope}_${basename(file, ".module.css").replace(/[^\w-]/g, "_")}`; + const result = await build({ + stdin: { contents: `export { default } from "scoped:${name}.module.css";`, loader: "js" }, + bundle: true, + write: false, + outdir: "/", + format: "cjs", + minifyWhitespace: true, + logLevel: "silent", + plugins: [ + { + name: "chord-css-module", + setup: (builder) => { + builder.onResolve({ filter: /^scoped:/ }, (args) => ({ + path: `/${args.path.slice(7)}`, + namespace: "scoped", + })); + // Kept as written: the facet's build resolves them relative to the module file. + builder.onResolve({ filter: /.*/ }, (args) => + args.kind === "url-token" || args.kind === "import-rule" + ? { path: args.path, external: true } + : undefined, + ); + builder.onLoad({ filter: /.*/, namespace: "scoped" }, () => ({ contents, loader: "local-css" })); + }, + }, + ], + }); + const css = result.outputFiles.find((output) => output.path.endsWith(".css"))?.text ?? ""; + const js = result.outputFiles.find((output) => output.path.endsWith(".js"))?.text ?? ""; + // esbuild's mapping module (an object of strings it wrote), run alone to read the mapping. + const module = { exports: {} as { default?: Record } }; + new Script(js).runInNewContext({ module, exports: module.exports }); + return { css, classes: { ...module.exports.default } }; +} diff --git a/packages/chord/src/next-node/plugins/evaluate.ts b/packages/chord/src/next-node/plugins/evaluate.ts index 5641fa251..50aaf4bb1 100644 --- a/packages/chord/src/next-node/plugins/evaluate.ts +++ b/packages/chord/src/next-node/plugins/evaluate.ts @@ -9,6 +9,7 @@ import { basename, isAbsolute, join, relative, resolve } from "node:path"; import { Script } from "node:vm"; import type { Principal } from "../../next/data/index.ts"; import type { Facet } from "../../next/host/index.ts"; +import { interop } from "../../next/surfaces/registered.ts"; import type { BuiltFacet } from "./build.ts"; import type { Plugin } from "./discover.ts"; import { registerScript } from "./stack.ts"; @@ -145,22 +146,3 @@ function fileOf(plugin: Plugin, path: string): string { } return file; } - -/** ESM namespaces wrapped for a CommonJS script, by namespace: one wrapper per namespace, held as long as it is. */ -const wrapped = new WeakMap(); - -/** - * A host package as a CommonJS script imports it: an ESM namespace wrapped once with `__esModule` (so esbuild's interop - * reads its `default` for a default import, its names for named ones), any other value as it is. - */ -function interop(value: unknown): unknown { - if (typeof value !== "object" || value === null || Object.prototype.toString.call(value) !== "[object Module]") { - return value; - } - let made = wrapped.get(value); - if (made === undefined) { - made = { ...value, __esModule: true }; - wrapped.set(value, made); - } - return made; -} diff --git a/packages/chord/src/next-node/plugins/runtime.ts b/packages/chord/src/next-node/plugins/runtime.ts index 85b8c3a3a..8b1751172 100644 --- a/packages/chord/src/next-node/plugins/runtime.ts +++ b/packages/chord/src/next-node/plugins/runtime.ts @@ -7,16 +7,9 @@ import { setFlagsFromString } from "node:v8"; import type { Context } from "../../context/index.ts"; -import { isChordError, type Principal } from "../../next/data/index.ts"; -import type { - CommittedReload, - Facet, - Host, - LoadReport, - PluginFacet, - PluginFailure, - ReloadReport, -} from "../../next/host/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import type { Host, LoadReport, PluginFacet, PluginFailure, ReloadReport } from "../../next/host/index.ts"; +import { LoadedFacets, type Prepared, type SetEntry } from "../../next/surfaces/loaded.ts"; import type { BuiltFacet } from "./build.ts"; import type { DiscoveryProblem, Plugin } from "./discover.ts"; import { evaluateFacet, type HostPackages } from "./evaluate.ts"; @@ -103,57 +96,19 @@ export function createPluginRuntime(): PluginRuntime { /** Whether this process turned V8's compilation cache off already. */ let cacheOff = false; -/** A plugin's facet as a host loaded it, with its script's digest. */ -type LoadedFacet = { readonly facet: Facet; readonly digest: string }; - -/** - * A host's plugins as the runtime loaded them: its core facets, its modules, each plugin's loaded facet, and its - * reloads, one after another (each computes its complete set once the previous one adopted its result). - */ -type HostPlugins = { - readonly core: readonly PluginFacet[]; - readonly modules: HostPackages; - readonly loaded: Map; - reloading: Promise; -}; - -/** A plugin's facet passed to a host, or why it was not: a build or evaluation failure. */ -type Prepared = - | { readonly kind: "facet"; readonly facet: Facet; readonly digest: string } - | { readonly kind: "failed"; readonly error: unknown }; - class Runtime implements PluginRuntime { - /** Each loaded host's plugins, held as long as the host is. */ - readonly #hosts = new WeakMap(); + /** Each loaded host's plugins and modules, held as long as the host is. */ + readonly #hosts = new WeakMap(); async load

(host: Host

, options: RuntimeLoad

, context: Context): Promise { - const { modules } = options; - const prepared = options.scripts.map( - ({ plugin, built }) => [plugin.name, prepare(plugin, built, modules)] as const, - ); - const failed: PluginFailure[] = []; - const facets: PluginFacet

[] = [...options.core]; - for (const [plugin, each] of prepared) { - if (each.kind === "failed") failed.push({ plugin, error: each.error }); - // Evaluated for this host's principal type: the facets are the app's (`Facet

` is invariant in `P`). - else facets.push({ name: plugin, facet: each.facet as unknown as Facet

}); - } - const report = await host.load(facets, context); - const left = new Set(report.failed.map(({ plugin }) => plugin)); - const loaded = new Map(); - for (const [plugin, each] of prepared) { - if (each.kind === "facet" && !left.has(plugin)) loaded.set(plugin, each); - } - // The host's own facets, for its later reloads. - const core = options.core as unknown as readonly PluginFacet[]; - this.#hosts.set(host, { core, modules, loaded, reloading: Promise.resolve() }); - const all = [...failed, ...report.failed]; - const required = new Set( - options.scripts.filter(({ plugin }) => plugin.required).map(({ plugin }) => plugin.name), - ); - const fatal = all.filter(({ plugin }) => required.has(plugin)); - if (fatal.length > 0) throw new PluginError(fatal, all); - return { failed: all }; + const { modules, scripts } = options; + const entries = scripts.map((script) => entryOf(script, modules)); + const { loaded, report } = await LoadedFacets.load(host, options.core, entries, context); + this.#hosts.set(host, { loaded, modules }); + const required = new Set(scripts.filter(({ plugin }) => plugin.required).map(({ plugin }) => plugin.name)); + const fatal = report.failed.filter(({ plugin }) => required.has(plugin)); + if (fatal.length > 0) throw new PluginError(fatal, report.failed); + return report; } reload

( @@ -163,60 +118,18 @@ class Runtime implements PluginRuntime { ): Promise { const state = this.#hosts.get(host); if (state === undefined) throw new Error("plugins.reload: the host was not loaded by this runtime"); - // One after another per host: a reload computing its set from a loaded map an earlier one is about to change - // would pass that plugin's old facet and roll it back. - const reload = state.reloading.then(() => this.#reload(host, state, scripts, context)); - // The tail settles to nothing: a report kept as its value would keep a rejected facet's closure (its error). - state.reloading = reload.then( - () => {}, - () => {}, - ); - return reload; + const entries = scripts.map((script) => entryOf(script, state.modules)); + return state.loaded.reload(host, entries, context); } +} - /** One reload of `host` with the complete set `scripts`. */ - async #reload

( - host: Host

, - state: HostPlugins, - scripts: readonly PluginScript[], - context: Context, - ): Promise { - const failed: PluginFailure[] = []; - const next = new Map(); - for (const { plugin, built } of scripts) { - const { name } = plugin; - const loaded = state.loaded.get(name); - if (built.kind === "script" && built.digest === loaded?.digest) { - next.set(name, loaded); - continue; - } - const prepared = prepare(plugin, built, state.modules); - if (prepared.kind === "facet") { - next.set(name, prepared); - continue; - } - failed.push({ plugin: name, error: prepared.error }); - if (loaded !== undefined) next.set(name, loaded); - } - const facets = [...state.core, ...[...next].map(([name, { facet }]) => ({ name, facet }))]; - // The host takes facets of its own principal type: the runtime loaded them for it. The plugins whose build or - // evaluation failed are the host's to report, and its closure rule's to consider. - let report: ReloadReport; - try { - report = await host.reload(facets as unknown as readonly PluginFacet

[], context, { failed }); - } catch (error) { - // Aborted after its commit: the commit stands, and is what the host runs now. - const committed = committedOf(error); - if (committed !== undefined) adopt(state, next, committed.failed); - throw error; - } - adopt( - state, - next, - report.failed.map(({ plugin }) => plugin), - ); - return report; - } +/** `script` as one plugin of a complete set: its digest, and its evaluation with `modules` when it is new. */ +function entryOf({ plugin, built }: PluginScript, modules: HostPackages): SetEntry { + return { + name: plugin.name, + digest: built.kind === "script" ? built.digest : undefined, + prepare: () => prepare(plugin, built, modules), + }; } /** `built` evaluated for `plugin` with `modules`, or why not: its build failed, or its evaluation threw. */ @@ -232,25 +145,3 @@ function prepare(plugin: Plugin, built: BuiltFacet, modules: HostPackages): Prep return { kind: "failed", error }; } } - -/** - * Adopts as `state`'s loaded facets the set `next` a reload committed: a plugin the host kept old (`failed`) stays loaded - * as it was, an added one it left out is not loaded. - */ -function adopt(state: HostPlugins, next: Map, failed: readonly string[]): void { - for (const plugin of failed) { - const loaded = state.loaded.get(plugin); - if (loaded === undefined) next.delete(plugin); - else next.set(plugin, loaded); - } - state.loaded.clear(); - for (const [name, each] of next) state.loaded.set(name, each); -} - -/** What the commit of a reload that rejected applied: an `aborted` error after its commit says (`details.committed`). */ -function committedOf(error: unknown): CommittedReload | undefined { - if (!isChordError(error, "aborted")) return undefined; - const details = error.details as { readonly committed?: CommittedReload } | null | undefined; - // The kernel's own error: its `committed` is a `CommittedReload`. - return typeof details === "object" && details !== null ? details.committed : undefined; -} diff --git a/packages/chord/src/next-node/surfaces/README.md b/packages/chord/src/next-node/surfaces/README.md new file mode 100644 index 000000000..3859e50fa --- /dev/null +++ b/packages/chord/src/next-node/surfaces/README.md @@ -0,0 +1,40 @@ +# surfaces (server side) + +The web and terminal surfaces' server side (slice 11 §1, §4; decisions "Slice 11, P3" and "P5"): +`@earendil-works/chord/next/web/server` once v2 has its entry point. The app kit (`startServer({ web, tui })`) creates +them; by hand: + +```ts +const web = await webSurface(server, { plugins, entry: "./src/web/main.ts", page: (request, assets) => shell(request.nonce, assets), paths: ["/"], public: false }, ctx); +const tui = await tuiSurface(server, { plugins, entry: "./src/tui/main.ts" }, ctx); +// web.facet, tui.facet among the server's core facets; web.step("web"), tui.step("tui") after the server's in a rollout +``` + +- **Base bundles** (`bundle.ts`): built at start by esbuild from the entry (its imports inlined once: the app's API for + the kind, Chord, React), named by the hash of all its outputs. A page's: an IIFE `base..js`, its CSS + `base..css`, assets under `assets/`. A terminal's: one CommonJS script the sandbox evaluates from memory with its own + `require` (the allowed builtins only). A broken base fails the start. +- **Published scripts** (`published.ts`, `current.ts`): the kind's current set (`plugins.scripts(kind)` at start, + `plugins.current(kind)` at each rollout step) as registered scripts `plugin/..js`, their + CSS beside them (each script with its inline source map), plugin assets under `assets/` (the kit's + `publicPath`), and the combined file `plugins..js` (without source maps: a browser reads one per file); + a plugin whose new build failed keeps its last script, CSS and assets (reported, stays dirty); the files of the last + four sets stay served. Terminal plugins have no assets (a Node build refuses them). The set goes to a local object (`chord.web.current`, a writable) the scripts object + derives from; a rollout step publishes without waiting for clients. +- **Files** (`served.ts`): `Cache-Control: private, max-age=31536000, immutable` (the baseline's `no-store` overridden + for these routes only). +- **A page's load:** the page, two scripts (the combined plugins file, the base bundle) and one stylesheet per plugin + that has CSS (beside the base bundle's own). +- **Web routes** (`web.ts`): `GET /_chord/web/:file`, `plugin/:file`, `assets/:file` (signed in unless `public`), `GET + /_chord/web/base` (public, `no-store`: an outdated page asks it), the page at `paths` (`page: true`, signed in unless + `public`). CSP additions: `style-src 'self'`, `img-src 'self' data:`, `font-src 'self'`. Scripts only from `/_chord/web/` (the HTTP component's `scripts`: `script-src` is that path of the request's origin + and the nonce, so no other same-origin response runs as a script). `PageAssets` (`bootstrap`, + `script`, `plugins`, `styles`) and `pageScripts(assets, nonce)` for the app's template. +- **Terminal routes** (`tui.ts`): `GET /_chord/tui/start` (`TuiStart`: base hash, script, combined; signed in), `GET + /_chord/tui/base` (public), the files (signed in). + +Examples: `56-web` (a page, two web plugins, an edit applied in place), `57-tui` (a sandboxed terminal client), +`58-themes` (a theme plugin restyling a page and a terminal live), `59-chat` (a chat: one worker per channel, uploads +through the kit's `Files`, a page and a terminal live in one room, pick-a-name login). + +Tests: `test/next-node/surfaces/`. diff --git a/packages/chord/src/next-node/surfaces/bundle.ts b/packages/chord/src/next-node/surfaces/bundle.ts new file mode 100644 index 000000000..c10a56387 --- /dev/null +++ b/packages/chord/src/next-node/surfaces/bundle.ts @@ -0,0 +1,71 @@ +// Base bundles (slice 11 §1, decision "Slice 11, P3"): a client kind's base code built on the server at start by +// esbuild from the app's entry (its start code, its API with the kind's module, Chord, React: inlined once), named by +// the hash of everything it is made of, served forever. A page's is a browser script with its stylesheet and assets; +// a terminal's one CommonJS script the sandbox evaluates with its own `require` (its allowed builtins only). + +import { createHash } from "node:crypto"; +import { basename, extname } from "node:path"; +import { build } from "esbuild"; +import { CSS, JAVASCRIPT, mimeOf, type ServedFile } from "./served.ts"; + +/** + * A built base bundle: its hash, its script's and stylesheet's paths (under the surface's prefix), and every file by + * path. + */ +export type BaseBundle = { + readonly hash: string; + readonly script: string; + readonly style: string | undefined; + readonly files: ReadonlyMap; +}; + +/** Where a base bundle runs: a page (a script, its CSS and assets), or a terminal's sandbox (one CommonJS script). */ +export type BasePlatform = "browser" | "sandbox"; + +/** + * Builds the base bundle of `entry` for `platform`, its files under `prefix` (`/_chord/web/`): the script + * `base..js`, the stylesheet `base..css` when it imports CSS, assets under `assets/` by their own hashes. + * Throws esbuild's failure: the server does not start with a broken base. + */ +export async function buildBase(entry: string, platform: BasePlatform, prefix: string): Promise { + const browser = platform === "browser"; + const result = await build({ + entryPoints: [entry], + outdir: "/", + entryNames: "base", + assetNames: "assets/[name]-[hash]", + publicPath: prefix, + write: false, + bundle: true, + format: browser ? "iife" : "cjs", + platform: browser ? "browser" : "node", + target: browser ? "es2022" : "node22", + minify: true, + legalComments: "none", + // Ignore any tsconfig.json: TypeScript is stripped, JSX compiled with the automatic React runtime. + tsconfigRaw: "{}", + jsx: "automatic", + define: { "process.env.NODE_ENV": '"production"' }, + logLevel: "silent", + }); + const hash = createHash("sha256"); + for (const output of [...result.outputFiles].sort((a, b) => (a.path < b.path ? -1 : 1))) { + hash.update(output.path).update(output.contents); + } + const digest = hash.digest("hex").slice(0, 16); + const files = new Map(); + let script = ""; + let style: string | undefined; + for (const output of result.outputFiles) { + const extension = extname(output.path); + if (basename(output.path) === `base${extension}`) { + const path = `${prefix}base.${digest}${extension}`; + if (extension === ".js") script = path; + else style = path; + files.set(path, { mime: extension === ".js" ? JAVASCRIPT : CSS, bytes: output.contents }); + } else { + files.set(`${prefix}assets/${basename(output.path)}`, { mime: mimeOf(output.path), bytes: output.contents }); + } + } + return { hash: digest, script, style, files }; +} diff --git a/packages/chord/src/next-node/surfaces/current.ts b/packages/chord/src/next-node/surfaces/current.ts new file mode 100644 index 000000000..0814c698b --- /dev/null +++ b/packages/chord/src/next-node/surfaces/current.ts @@ -0,0 +1,70 @@ +// The scripts objects a surface serves (slice 11 §1, decision "Slice 11, P5"): on the server, one local object per +// surface holds the published set (a writable, set by each publish), and the surface's scripts object (`WebScripts`, +// `TuiScripts`), made per principal, shows it. A rollout publishes without waiting for clients: each client's hold gets +// one frame, and a reconnect's snapshot brings the set it missed. + +import type { Context } from "../../context/index.ts"; +import { local, method } from "../../next/contracts/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet, type Host } from "../../next/host/index.ts"; +import { ScriptSet, type ScriptsContract, TuiScripts, WebScripts } from "../../next/surfaces/index.ts"; + +/** A surface: the web (pages) or the terminal. */ +export type SurfaceName = "web" | "tui"; + +/** The local object holding a surface's published set, by surface. */ +const CURRENT = { + web: currentContract("chord.web.current"), + tui: currentContract("chord.tui.current"), +}; + +/** The scripts object each surface serves. */ +const SCRIPTS: Readonly> = { web: WebScripts, tui: TuiScripts }; + +/** The contract of a surface's published set: its `set` and `publish`. */ +function currentContract(name: N) { + return local(name, { version: 1, params: {}, members: { set: ScriptSet, publish: method(ScriptSet) } }); +} + +/** + * The surface's facet serving its published set (starting at `published.set`, the set taken before the load) and its + * scripts object for every principal that may hold it. + */ +export function scriptsFacet

( + surface: SurfaceName, + published: { readonly set: ScriptSet }, +): Facet

{ + const Current = CURRENT[surface]; + return defineFacet

((host) => { + host.serve(Current, (_params, self) => { + // The latest set: an attempt after a failure starts at what was published last. + const set = self.writable(published.set); + return { + set: self.derive([set], ([value]) => value), + publish: async (next) => { + set.set(next); + return null; + }, + }; + }); + host.serve(SCRIPTS[surface], (_params, _principal, self) => { + const current = self.get(Current, {}); + return { variant: "main", members: { set: self.derive([current.set], ([value]) => value) } }; + }); + }); +} + +/** Publishes `set` as `surface`'s current set on `host` (loaded with `scriptsFacet`). */ +export async function publishTo

( + host: Host

, + surface: SurfaceName, + set: ScriptSet, + context: Context, +): Promise { + const current = await host.get(CURRENT[surface], {}, context); + try { + await current.publish(set, context); + } finally { + current.release(); + } +} diff --git a/packages/chord/src/next-node/surfaces/index.ts b/packages/chord/src/next-node/surfaces/index.ts new file mode 100644 index 000000000..186bce171 --- /dev/null +++ b/packages/chord/src/next-node/surfaces/index.ts @@ -0,0 +1,14 @@ +// Chord v2's client surfaces, the server side (`@earendil-works/chord/next/web/server` once v2 has its entry point; +// slice 11 §1, §4): the web surface (base bundle, plugin scripts, `WebScripts`, the page) and the terminal surface. + +export type { BaseBundle } from "./bundle.ts"; +export type { ServedFile } from "./served.ts"; +export { TUI_PREFIX, type TuiStart, type TuiSurface, type TuiSurfaceOptions, tuiSurface } from "./tui.ts"; +export { + type PageAssets, + pageScripts, + WEB_PREFIX, + type WebSurface, + type WebSurfaceOptions, + webSurface, +} from "./web.ts"; diff --git a/packages/chord/src/next-node/surfaces/published.ts b/packages/chord/src/next-node/surfaces/published.ts new file mode 100644 index 000000000..c868e8b40 --- /dev/null +++ b/packages/chord/src/next-node/surfaces/published.ts @@ -0,0 +1,159 @@ +// A client kind's published scripts (slice 11 §1, decisions "Slice 11, P3" and "P5"): what the server serves of the +// kind's plugins, as files under the surface's prefix, and the set its scripts object shows. Each plugin's facet is a +// script that registers itself (`plugin/..js`, its stylesheet beside it), the whole set one +// combined file named by its hash (`plugins..js`), assets under `assets/`; all named by their +// content, served forever. A rollout shipping the kind publishes a new set (a plugin whose new build failed keeps its +// last script) without waiting for clients; the files of the last few sets stay served for clients that still load +// them. + +import { createHash } from "node:crypto"; +import type { Context } from "../../context/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import type { Host, PluginFailure } from "../../next/host/index.ts"; +import { registeredSource, type ScriptSet } from "../../next/surfaces/index.ts"; +import type { Built, PluginScript, Plugins, RolloutStep } from "../plugins/index.ts"; +import { publishTo, type SurfaceName } from "./current.ts"; +import { CSS, JAVASCRIPT, mimeOf, type ServedFile } from "./served.ts"; + +/** How many sets' files stay served: the current one and the few before it, for clients loading them meanwhile. */ +const KEPT_SETS = 4; + +/** One plugin's published script: its digest, its registered text, its stylesheet. */ +/** + * One plugin's published script: its digest, its registered text, its stylesheet and assets (kept with the script: a + * plugin whose next build fails keeps all three). + */ +type Published = { + readonly digest: string; + readonly text: string; + readonly css: string | undefined; + readonly assets: readonly { readonly name: string; readonly bytes: Uint8Array }[]; +}; + +/** A script's inline source map comment. */ +const SOURCE_MAP = /\n\/\/# sourceMappingURL=data:[^\n]*/g; + +/** A client kind's published scripts. */ +export class PublishedScripts

{ + readonly #surface: SurfaceName; + readonly #kind: string; + readonly #prefix: string; + readonly #base: string; + readonly #plugins: Plugins; + readonly #host: Host

; + /** The published script of each plugin now. */ + readonly #current = new Map(); + /** The files of the last `KEPT_SETS` sets, newest last, each by path. */ + readonly #sets: ReadonlyMap[] = []; + #set: ScriptSet | undefined; + + constructor(surface: SurfaceName, kind: string, prefix: string, base: string, plugins: Plugins, host: Host

) { + this.#surface = surface; + this.#kind = kind; + this.#prefix = prefix; + this.#base = base; + this.#plugins = plugins; + this.#host = host; + } + + /** The set published now (after `start`). */ + get set(): ScriptSet { + return this.#set!; + } + + /** + * Takes the kind's current set as the first one (its plugins' facets of the kind built on first use), before the + * server's load (the scripts object starts with it). The plugins whose build failed. + */ + async start(context: Context): Promise { + return this.#publish(await this.#plugins.scripts(this.#kind, context)); + } + + /** A published file by its path, from the last few sets. */ + file(path: string): ServedFile | undefined { + for (let index = this.#sets.length - 1; index >= 0; index--) { + const found = this.#sets[index]!.get(path); + if (found !== undefined) return found; + } + return undefined; + } + + /** + * The rollout step publishing the kind's current set as the rollout applied its build: one "host" named `name`, its + * report the plugins whose new build failed (they keep their last script, and stay dirty). Clients are not waited for. + */ + step(name: string): RolloutStep { + return { + name, + kinds: [this.#kind], + reload: async (_built: Built, context: Context) => { + const before = new Map([...this.#current].map(([plugin, { digest }]) => [plugin, digest])); + const failed = this.#publish(this.#plugins.current(this.#kind) ?? []); + await publishTo(this.#host, this.#surface, this.set, context); + const after = [...this.#current]; + const plugins = { + added: after.filter(([plugin]) => !before.has(plugin)).map(([plugin]) => plugin), + changed: after + .filter(([plugin, { digest }]) => before.has(plugin) && before.get(plugin) !== digest) + .map(([plugin]) => plugin), + removed: [...before.keys()].filter((plugin) => !this.#current.has(plugin)), + }; + return { hosts: [{ host: name, report: { plugins, failed, objects: [] } }] }; + }, + }; + } + + /** + * Takes `scripts` (the kind's complete set) as the current one: a failed build keeps its plugin's last script; its + * files kept with the last few sets'. The plugins whose build failed. + */ + #publish(scripts: readonly PluginScript[]): readonly PluginFailure[] { + const failed: PluginFailure[] = []; + const next = new Map(); + for (const { plugin, built } of scripts) { + if (built.kind === "script") { + const text = registeredSource(plugin.name, built.digest, built.source); + next.set(plugin.name, { digest: built.digest, text, css: built.css?.text, assets: built.assets }); + continue; + } + failed.push({ plugin: plugin.name, error: new Error(built.errors.join("\n")) }); + const kept = this.#current.get(plugin.name); + if (kept !== undefined) next.set(plugin.name, kept); + } + this.#current.clear(); + for (const [name, each] of [...next].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))) + this.#current.set(name, each); + const { set, files } = this.#files(); + this.#sets.push(files); + if (this.#sets.length > KEPT_SETS) this.#sets.shift(); + this.#set = set; + return failed; + } + + /** + * The set of the current scripts and its files: each script (its source map inline), stylesheet and asset, and the + * combined file (without source maps: a browser reads only one per file, so they would be bytes for nothing). + */ + #files(): { set: ScriptSet; files: Map } { + const files = new Map(); + const entries: ScriptSet["plugins"][number][] = []; + const combined: string[] = []; + for (const [name, { digest, text, css, assets }] of this.#current) { + // A plugin name (`@scope/name`) as one path segment, its slash a tilde: the set's URLs are looked up as they + // are, never parsed back. + const stem = `${this.#prefix}plugin/${name.replace("/", "~")}.${digest}`; + files.set(`${stem}.js`, { mime: JAVASCRIPT, bytes: new TextEncoder().encode(text) }); + if (css !== undefined) files.set(`${stem}.css`, { mime: CSS, bytes: new TextEncoder().encode(css) }); + entries.push({ name, digest, url: `${stem}.js`, ...(css === undefined ? {} : { css: `${stem}.css` }) }); + combined.push(text.replace(SOURCE_MAP, "")); + for (const asset of assets) { + files.set(`${this.#prefix}assets/${asset.name}`, { mime: mimeOf(asset.name), bytes: asset.bytes }); + } + } + const body = combined.join(""); + const hash = createHash("sha256").update(body).digest("hex").slice(0, 16); + const url = `${this.#prefix}plugins.${hash}.js`; + files.set(url, { mime: JAVASCRIPT, bytes: new TextEncoder().encode(body) }); + return { set: { base: this.#base, combined: url, plugins: entries }, files }; + } +} diff --git a/packages/chord/src/next-node/surfaces/served.ts b/packages/chord/src/next-node/surfaces/served.ts new file mode 100644 index 000000000..142b1b8b3 --- /dev/null +++ b/packages/chord/src/next-node/surfaces/served.ts @@ -0,0 +1,50 @@ +// The files the surfaces serve (slice 11 §1, §4): base bundles, plugin scripts, stylesheets and assets, each named by +// its content and so cached by clients forever; their types. + +import { extname } from "node:path"; +import type { HttpResponse } from "../http/index.ts"; + +/** A file the server serves: its type and bytes. */ +export type ServedFile = { readonly mime: string; readonly bytes: Uint8Array }; + +/** A served file's response, cached by the browser forever (its path names its content); 404 for none. */ +export function served(found: ServedFile | undefined): HttpResponse { + if (found === undefined) return { kind: "error", status: 404 }; + const { mime, bytes } = found; + return { + kind: "bytes", + mime, + size: bytes.length, + read: async (offset, length) => bytes.subarray(offset, offset + length), + // Overrides the baseline's `no-store` for these files only. + headers: { "Cache-Control": "private, max-age=31536000, immutable" }, + }; +} + +/** The type of a script. */ +export const JAVASCRIPT = "text/javascript; charset=utf-8"; + +/** The type of a stylesheet. */ +export const CSS = "text/css; charset=utf-8"; + +/** The type of a built asset, by extension. */ +export function mimeOf(path: string): string { + return ASSET_TYPES[extname(path).slice(1).toLowerCase()] ?? "application/octet-stream"; +} + +/** The types of the assets builds collect, by extension. */ +const ASSET_TYPES: Readonly> = { + svg: "image/svg+xml", + png: "image/png", + jpg: "image/jpeg", + jpeg: "image/jpeg", + gif: "image/gif", + webp: "image/webp", + woff: "font/woff", + woff2: "font/woff2", + ttf: "font/ttf", + wasm: "application/wasm", + json: "application/json", + css: CSS, + js: JAVASCRIPT, +}; diff --git a/packages/chord/src/next-node/surfaces/tui.ts b/packages/chord/src/next-node/surfaces/tui.ts new file mode 100644 index 000000000..ddb309c4f --- /dev/null +++ b/packages/chord/src/next-node/surfaces/tui.ts @@ -0,0 +1,67 @@ +// The terminal surface's server side (slice 11 §4, decisions "Slice 11, P3" and "P8"): the terminal kind's base bundle +// (one ES module the sandbox evaluates) built from the app's terminal entry at start, the kind's plugin scripts +// published as registered scripts and one combined file, `TuiScripts` showing the current set, all served to the +// launcher, which signs in with a token. Routes and objects are `Http` contributions of the surface's facet. + +import type { Context } from "../../context/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet, type Host } from "../../next/host/index.ts"; +import { Http, type HttpRequest, type HttpResponse } from "../http/index.ts"; +import type { Plugins, RolloutStep } from "../plugins/index.ts"; +import { buildBase } from "./bundle.ts"; +import { scriptsFacet } from "./current.ts"; +import { PublishedScripts } from "./published.ts"; +import { type ServedFile, served } from "./served.ts"; + +/** What `tuiSurface` takes. */ +export type TuiSurfaceOptions = { + /** The terminal facet kind: `tui` by default. */ + readonly kind?: string; + readonly plugins: Plugins; + /** The app's terminal start code (it calls `startTerminalRuntime`): the base bundle's entry. */ + readonly entry: string; +}; + +/** What a launcher downloads at its start (`GET /_chord/tui/start`): the base bundle and the combined plugin file. */ +export type TuiStart = { readonly base: string; readonly script: string; readonly combined: string }; + +/** The terminal surface: its facet (for the server's core facets), its rollout step, its base bundle's hash. */ +export type TuiSurface

= { + readonly facet: Facet

; + step(name: string): RolloutStep; + readonly base: string; +}; + +/** Where the terminal surface's files live. */ +export const TUI_PREFIX = "/_chord/tui/"; + +/** Builds the terminal's base bundle and takes the kind's current set, then returns the surface. */ +export async function tuiSurface

( + server: Host

, + options: TuiSurfaceOptions, + context: Context, +): Promise> { + const base = await buildBase(options.entry, "sandbox", TUI_PREFIX); + const published = new PublishedScripts("tui", options.kind ?? "tui", TUI_PREFIX, base.hash, options.plugins, server); + await published.start(context); + const file = (path: string): ServedFile | undefined => base.files.get(path) ?? published.file(path); + const serve = async (request: HttpRequest): Promise => served(file(request.path)); + const facet = defineFacet

((host, self) => { + scriptsFacet

("tui", published).setup(host, self); + self.contribute(Http, { + routes: { + [`GET ${TUI_PREFIX}start`]: async () => { + const start: TuiStart = { base: base.hash, script: base.script, combined: published.set.combined }; + return { kind: "json", value: start, headers: { "Cache-Control": "no-store" } }; + }, + [`GET ${TUI_PREFIX}base`]: { + public: true, + handler: async () => ({ kind: "json", value: base.hash, headers: { "Cache-Control": "no-store" } }), + }, + [`GET ${TUI_PREFIX}:file`]: serve, + [`GET ${TUI_PREFIX}plugin/:file`]: serve, + }, + }); + }); + return { facet, step: (name) => published.step(name), base: base.hash }; +} diff --git a/packages/chord/src/next-node/surfaces/web.ts b/packages/chord/src/next-node/surfaces/web.ts new file mode 100644 index 000000000..87cb8dca1 --- /dev/null +++ b/packages/chord/src/next-node/surfaces/web.ts @@ -0,0 +1,137 @@ +// The web surface's server side (slice 11 §1, decisions "Slice 11's points" and "P3" to "P5"): the base bundle built +// from the app's entry at start, the plugins' web scripts published as registered scripts and one combined file, +// `WebScripts` showing the current set, and the page through the app's own template; everything as `Http` +// contributions of the surface's facet, which the app puts among the server's core facets. The app keeps its HTML, +// CSS, components and themes. + +import type { Context } from "../../context/index.ts"; +import type { Principal } from "../../next/data/index.ts"; +import { defineFacet, type Facet, type Host } from "../../next/host/index.ts"; +import { REGISTER, type ScriptSet } from "../../next/surfaces/index.ts"; +import { Http, type HttpRequest, type HttpResponse, type HttpRouteOptions } from "../http/index.ts"; +import type { Plugins, RolloutStep } from "../plugins/index.ts"; +import { type BaseBundle, buildBase } from "./bundle.ts"; +import { scriptsFacet } from "./current.ts"; +import { PublishedScripts } from "./published.ts"; +import { type ServedFile, served } from "./served.ts"; + +/** What `webSurface` takes. */ +export type WebSurfaceOptions = { + /** The web facet kind: `web` by default. */ + readonly kind?: string; + readonly plugins: Plugins; + /** The app's web start code (it calls `startPage`): the base bundle's entry. */ + readonly entry: string; + /** The page's HTML: the app's template, given the request (its `nonce`) and the assets to include. */ + readonly page: (request: HttpRequest, assets: PageAssets) => string | Promise; + /** The paths serving the page: `["/"]` by default (patterns, `/*` for a page with its own routing). */ + readonly paths?: readonly string[]; + /** Whether the page and its scripts are served without a principal (a public app); signed in only by default. */ + readonly public?: boolean; +}; + +/** + * What a page includes, in this order: `bootstrap` as an inline script with the response's nonce, then the scripts + * `plugins` and `script` (deferred, in that order), and the stylesheets `styles` (the base bundle's first, then each + * plugin's). `pageScripts` writes the script tags. + */ +export type PageAssets = { + readonly bootstrap: string; + readonly script: string; + readonly plugins: string; + readonly styles: readonly string[]; +}; + +/** The web surface: its facet (for the server's core facets), its rollout step, its base bundle's hash. */ +export type WebSurface

= { + readonly facet: Facet

; + /** The rollout step publishing the web kind's current set (after the server's step). */ + step(name: string): RolloutStep; + readonly base: string; +}; + +/** Where the web surface's files live. */ +export const WEB_PREFIX = "/_chord/web/"; + +/** + * Builds the base bundle and takes the web kind's current set (its facets built on first use), then returns the + * surface. Rejects with esbuild's failure when the base bundle does not build. + */ +export async function webSurface

( + server: Host

, + options: WebSurfaceOptions, + context: Context, +): Promise> { + const base = await buildBase(options.entry, "browser", WEB_PREFIX); + const published = new PublishedScripts("web", options.kind ?? "web", WEB_PREFIX, base.hash, options.plugins, server); + await published.start(context); + const file = (path: string): ServedFile | undefined => base.files.get(path) ?? published.file(path); + const facet = defineFacet

((host, self) => { + scriptsFacet

("web", published).setup(host, self); + self.contribute(Http, { + routes: webRoutes(options, base.hash, file, () => pageAssets(base, published)), + csp: { "style-src": ["'self'"], "img-src": ["'self'", "data:"], "font-src": ["'self'"] }, + // Scripts from the surface's files only (and the page's nonce): no other same-origin response runs (Opus L8). + scripts: [WEB_PREFIX], + }); + }); + return { facet, step: (name) => published.step(name), base: base.hash }; +} + +/** The surface's routes: its files (served forever), its current base hash (public), and the page. */ +function webRoutes( + options: WebSurfaceOptions, + base: string, + file: (path: string) => ServedFile | undefined, + assets: () => PageAssets, +): Record { + const access = options.public === true ? { public: true as const } : {}; + const serve = async (request: HttpRequest): Promise => served(file(request.path)); + const routes: Record = { + [`GET ${WEB_PREFIX}:file`]: { ...access, handler: serve }, + [`GET ${WEB_PREFIX}plugin/:file`]: { ...access, handler: serve }, + [`GET ${WEB_PREFIX}assets/:file`]: { ...access, handler: serve }, + // Asked by a page before it reconnects: another base means its code is outdated. Public: an outdated page whose + // session ended still learns it should reload. + [`GET ${WEB_PREFIX}base`]: { + public: true, + handler: async () => ({ kind: "json", value: base, headers: { "Cache-Control": "no-store" } }), + }, + }; + for (const path of options.paths ?? ["/"]) { + routes[`GET ${path}`] = { + ...access, + page: true, + handler: async (request) => ({ kind: "html", body: await options.page(request, assets()) }), + }; + } + return routes; +} + +/** What the page includes now: the bootstrap, the base script, the current combined file, the stylesheets. */ +function pageAssets(base: BaseBundle, published: { readonly set: ScriptSet }): PageAssets { + const { set } = published; + const styles = [ + ...(base.style === undefined ? [] : [base.style]), + ...set.plugins.flatMap(({ css }) => (css === undefined ? [] : [css])), + ]; + return { bootstrap: bootstrap(base.hash), script: base.script, plugins: set.combined, styles }; +} + +/** + * The page's inline bootstrap: the base hash for `startPage`, and the `chordFacet` the combined file's scripts call + * before the base bundle runs (they queue; `startPage` takes them). + */ +function bootstrap(hash: string): string { + return `globalThis.chordPage={base:${JSON.stringify(hash)},queued:[]};globalThis.${REGISTER}=function(p,d,f){chordPage.queued.push([p,d,f])};`; +} + +/** The page's script tags for `assets`: the inline bootstrap with `nonce`, then the deferred scripts; the stylesheets. */ +export function pageScripts(assets: PageAssets, nonce: string): string { + return [ + ...assets.styles.map((href) => ``), + ``, + ``, + ``, + ].join("\n"); +} diff --git a/packages/chord/src/next-node/tui/README.md b/packages/chord/src/next-node/tui/README.md new file mode 100644 index 000000000..79803bf19 --- /dev/null +++ b/packages/chord/src/next-node/tui/README.md @@ -0,0 +1,75 @@ +# tui (launcher) + +The trusted side of a terminal client (slice 11 §4, decision "Slice 11, P8"; p2's `tui-client.ts`, `tui-sandbox.ts`, +`tui-bootstrap.mjs`, `tui-filter.ts`): `@earendil-works/chord/next/tui/launcher` once v2 has its entry point. Needs Node +26.5.1 or later (the Node its sandbox was checked on); `requireSandboxNode` refuses older ones with a clear error. + +```ts +const terminal = await startTerminal({ + url: "http://127.0.0.1:8788", + credentials: { kind: "file", path: "~/.acme/owner-token" }, // or { kind: "token", token }, { kind: "pairing", label, save? } + mode: "dark", colors: "truecolor", // the terminal's background and colours +}); +process.exitCode = await terminal.exited; +``` + +- **Credentials** (`credentials.ts`): the launcher keeps the token; pairing asks for a code, shows it, polls every 2 s + until the owner approves it, and may save the token (created, mode 0600). +- **The sandbox** (`sandbox.ts`): a Node child process of the launcher's binary, in two layers: + - the operating system's: stdin, stdout and stderr pipes to the launcher (no descriptor of the child is a terminal), + an empty environment (but `LD_LIBRARY_PATH` when the launcher has one: it needed it to run the same binary), a + private directory holding only the bootstrap; `--permission` allowing only reading the bootstrap's file (no other + file, no network, child processes, workers, addons, inspector or WASI), `--no-experimental-sqlite` (SQLite opens + files past the permission model), a 96 MiB heap. The bootstrap is a file, not `-e`, which puts every builtin on + `globalThis`; + - JavaScript's, a second layer: only the bootstrap evaluates downloaded code (the base bundle, at most 8 MiB, a + CommonJS script; the plugins' registered scripts through the bridge's `evaluate`), with `vm.Script` (options of no + prototype, no dynamic import callback: `import()` throws) and a `require` (and `process.getBuiltinModule`) + answering a short list of builtins that compute in memory (`SANDBOX_BUILTINS`, looked up in an object of no + prototype); `process` is a frozen plain object with `SANDBOX_PROCESS` only; the bootstrap keeps its own references + to what it uses, and Node's `process` has its listener calls as own properties (changing the shared + `EventEmitter.prototype` catches nothing: Opus E2); every prototype of Node's `process`, `Object.prototype` + included, takes no new property and keeps its own ones (a getter for a property Node reads on its process, such as + `noDeprecation`, would get the real one: Opus E7); + - messages both ways as length-prefixed binary frames (`pipe.ts`: a header as JSON, the text or bytes as body), each + at most 17 MiB; the launcher checks every message and ends a sandbox sending a malformed one or not reading; + - known gaps, each needing the JavaScript layer broken first unless noted: the child may signal the user's processes + (Node has no permission for signals; the launcher ignores SIGUSR1, which would open its inspector: Opus E8), read + the machine's details (`os`), and learn whether any path exists (`require.resolve` fails one way for a file that + exists, another for one that does not: Opus E13, a Node gap); memory outside its heap has no limit (only the child + dies). +- **The launcher** (`launcher.ts`): downloads `/_chord/tui/start`, the base bundle and the combined file with the token + (each body read up to 8 MiB, no further); gives the sandbox input (raw mode on a TTY) and resizes; writes its frames + through the output filter; opens and relays its link sockets to `/chord` (the token in the upgrade's header; at most + four at once); answers its fetches of terminal files only (`terminalFile`: the path resolved first, then + `/_chord/tui/` or `/_chord/tui/plugin/` of the server's origin); starts a new sandbox only when the server's + base bundle is another than the running one; prints the server's text (a pairing code, which must be a plain code) + through the filter; at its end resets the terminal's modes (sync, mouse, focus, paste, cursor, alternate screen, + colours, scroll region, autowrap), also when its process exits without that end; answers at most eight requests of + the sandbox at once (more are refused at once); ignores SIGUSR1 while it runs (no inspector opened by a signal). + Nothing the sandbox sends ends the launcher: a send on a socket not yet open is dropped, a socket with 16 MiB waiting + is closed, a message it fails on ends the client. +- **The output filter** (`filter.ts`): text and drawing sequences pass (cursor moves, erasing, SGR colours, the alternate + screen, bracketed paste, sync, mouse reporting); OSC (the clipboard among them), DCS and other control strings, + other modes and ESC sequences are dropped; an unterminated sequence ends with its frame; frames above 64 KiB are + dropped (the sandbox splits a large write into frames between sequences: `next-tui/frames.ts`). +- **Actions** (`actions.ts`), each confirmed on the launcher's prompt (the bottom lines, black on white after a reset + of all attributes, whatever the sandbox left; `y`): one at a time (a second request is refused at once); from the + request to the action's end the sandbox's frames are held (1 MiB at most); while the prompt is open its keys go to + it, and those in its first 300 ms are ignored (typed for the sandbox); a synchronized update the sandbox began is + closed and the prompt drawn last (again after a resize); its lines are wrapped to the terminal's width, never cut (a + prompt taller than the terminal is refused). Afterwards the held frames are written and the sandbox asked to redraw. + `copy` shows the text (five rows, then how many more characters) and refuses control and bidirectional characters (a + newline and a tab aside), then the launcher writes OSC 52. `open` takes http and https URLs without user info, names + the host first, and opens with `open`, `xdg-open` or `rundll32 url.dll,FileProtocolHandler` (as the coding agent), + never through a shell. `attach` refuses a path with control, format or unusual whitespace characters (on Windows + also UNC, device and `//host` paths and reserved device names), shows the path resolved by its text (its name, then + its directory) without looking at the machine, after a yes resolves its links and asks again when they lead + elsewhere, then opens it without blocking (no wait on a FIFO) and reads it only when it is the file confirmed (its + device and inode as when the last prompt was shown: no parent directory swapped for a link meanwhile) and a regular + file of at most 16 MiB, at the size its open handle had; one refusal for anything else. While a prompt is open, + Ctrl-C ends the client. +- **The way out:** three Ctrl-C within a second always end the client (exit code 130, the terminal reset), whatever the + sandbox does; each still reaches the sandbox, so an app that handles Ctrl-C itself works as before. + +Tests: `test/next-node/tui/` (the launcher and sandbox ones under `CHORD_TUI_NODE` or `node26`; skipped without). diff --git a/packages/chord/src/next-node/tui/actions.ts b/packages/chord/src/next-node/tui/actions.ts new file mode 100644 index 000000000..491a8f10d --- /dev/null +++ b/packages/chord/src/next-node/tui/actions.ts @@ -0,0 +1,212 @@ +// The launcher's actions (slice 11 §4; fix rounds 1 to 4: Fable's H1, H2, M1, L4, L6 and Opus's M1, M3, L2, L3, E5, +// E6, E9, E12, R1): what the sandbox may ask of the machine, each confirmed by the user on the launcher's own prompt +// before it touches it. Every line the prompt shows is wrapped to the terminal's width, never cut. Copying writes the +// text to the terminal's clipboard (OSC 52, which the output filter refuses to sandboxed code) and shows it first (up +// to five rows, then how many more characters); text with control characters (a newline and a tab aside) or +// bidirectional controls is refused. Opening a URL (http or https, no user info) names its host first and runs the +// platform's opener without a shell (as the coding agent's `open-browser.ts`). Attaching refuses a path with control, +// format or unusual whitespace characters (and on Windows UNC and device paths), shows the path resolved by its text +// (its name on its own line) without looking at the machine, after the user's yes resolves its links and asks again +// when they lead elsewhere, and reads the very file confirmed (its device and inode), a regular file of at most 16 MiB, +// exactly at the size its open handle has. + +import { spawn } from "node:child_process"; +import { constants } from "node:fs"; +import { open, realpath, stat } from "node:fs/promises"; +import { homedir } from "node:os"; +import { posix, win32 } from "node:path"; +import type { BridgeRequest } from "../../next-tui/bridge.ts"; + +/** + * The launcher's prompt on the real terminal, `columns` wide: `ask` shows `lines` (the question last) and resolves + * with the user's answer, yes or no; `write` writes to the terminal past the output filter (the launcher's own + * sequences). + */ +export type PromptTerminal = { + readonly columns: number; + ask(lines: readonly string[]): Promise; + write(data: string): void; +}; + +/** The prompt's rows for `lines` at `columns` wide: control characters removed, each line wrapped, never cut. */ +export function promptLines(lines: readonly string[], columns: number): string[] { + return lines.flatMap((line) => wrap(line, promptWidth(columns))); +} + +/** The text width of a prompt row (one column of padding each side). */ +function promptWidth(columns: number): number { + return Math.max(10, columns - 2); +} + +/** + * `line` with control characters removed, in rows of at most `width` columns, counting every character but printable + * ASCII as two (wide characters never push text past the edge, where the next row would cover it). + */ +export function wrap(line: string, width: number): string[] { + const rows: string[] = []; + let row = ""; + let used = 0; + for (const character of visible(line)) { + const cells = character >= " " && character <= "~" ? 1 : 2; + if (used + cells > width) { + rows.push(row); + row = ""; + used = 0; + } + row += character; + used += cells; + } + rows.push(row); + return rows; +} + +/** How many rows of a copied text the prompt shows before "and N more characters". */ +const SHOWN_ROWS = 5; + +/** + * Does the sandbox's `request` (`copy`, `open`) once the user confirmed it: whether it was done. Throws (the request + * refused) for text it will not copy and URLs it will not open. `opener` opens a URL (the platform's by default). + */ +export async function act( + request: Extract, + terminal: PromptTerminal, + opener: (url: string) => Promise = platformOpener, +): Promise { + if (request.kind === "copy") { + const { text } = request; + // Control characters (but newlines and tabs) and bidirectional controls could hide what a paste runs. + if (/[\u0000-\u0008\u000b-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/.test(text)) { + throw new Error("copy: the text holds control characters"); + } + const lines = text.split("\n"); + // Wrapped, never cut: a long line's end is where a pasted command would hide (Opus E6). + const rows = lines.flatMap((line) => wrap(line, promptWidth(terminal.columns) - 2)); + const hidden = rows.slice(SHOWN_ROWS).reduce((sum, row) => sum + row.length, 0); + const question = `Copy these ${text.length} characters (${lines.length} line${lines.length === 1 ? "" : "s"})? [y/N]`; + const shown = [ + ...rows.slice(0, SHOWN_ROWS).map((row) => ` ${row}`), + ...(hidden > 0 ? [` … and ${hidden} more characters`] : []), + ]; + if (!(await terminal.ask([...shown, question]))) return false; + terminal.write(`\x1b]52;c;${Buffer.from(text).toString("base64")}\x07`); + return true; + } + const url = openable(request.url); + if (url === undefined) throw new Error("open: only http and https URLs without user info open"); + if (!(await terminal.ask([` ${url.href}`, `Open ${url.host} in the browser? [y/N]`]))) return false; + await opener(url.href); + return true; +} + +/** The largest file the sandbox may be given: 16 MiB. */ +export const MAX_ATTACHMENT = 16 * 1024 * 1024; + +/** + * Reads the file at `path` (as the sandbox gave it; `~/` the home directory) for the sandbox once the user confirmed + * it: its name and bytes, or `null` when declined. Refused at once: a path with control, format or unusual whitespace + * characters, and on Windows a UNC, device or `//host` path or a reserved device name. Nothing on the machine is looked + * at before the user's yes (Opus E9: on Windows a lookup of `\\host\share` alone sends the user's credentials): the + * prompt shows the path resolved by its text (`~`, `.` and `..`); after the yes its links are resolved, and a path + * that leads elsewhere is asked about again, showing where; what is opened must be the file confirmed (its device and + * inode as when the last prompt was shown). Then it is opened without blocking (a FIFO waits for no + * writer: Opus E12) and read only when it is a regular file of at most `MAX_ATTACHMENT` bytes; one refusal for anything + * else (no hint what is there). + */ +export async function attach( + path: string, + terminal: PromptTerminal, + platform: NodeJS.Platform = process.platform, +): Promise<{ readonly name: string; readonly bytes: Uint8Array } | null> { + if (!attachable(path, platform)) throw cannotAttach(); + const paths = platform === "win32" ? win32 : posix; + const home = /^~(?:$|[\\/])/.test(path) ? paths.join(homedir(), path.slice(1)) : path; + const named = paths.resolve(home); + if (!(await terminal.ask(fileLines(named, paths, "Attach this file? [y/N]")))) return null; + const file = await realpath(named).catch(() => undefined); + if (file === undefined) throw cannotAttach(); + // The file the user confirms, by identity: what is read must be it (Opus R1: a parent directory swapped for a link + // after the yes would lead elsewhere, past `O_NOFOLLOW`, which guards the last step only). + const shown = await stat(file, { bigint: true }).catch(() => undefined); + if (shown === undefined) throw cannotAttach(); + if (file !== named && !(await terminal.ask(fileLines(file, paths, "It links here. Attach this file? [y/N]")))) { + return null; + } + // No waiting on a FIFO or device, and no link swapped in at the last step since the user saw the path. + const flags = constants.O_RDONLY | (constants.O_NONBLOCK ?? 0) | (constants.O_NOFOLLOW ?? 0); + const handle = await open(file, flags).catch(() => undefined); + if (handle === undefined) throw cannotAttach(); + try { + const info = await handle.stat({ bigint: true }); + if (info.dev !== shown.dev || info.ino !== shown.ino) throw cannotAttach(); + if (!info.isFile() || info.size > BigInt(MAX_ATTACHMENT)) throw cannotAttach(); + // Exactly its size now: a file growing meanwhile is read up to what was checked. + const bytes = new Uint8Array(Number(info.size)); + let read = 0; + while (read < bytes.length) { + const { bytesRead } = await handle.read(bytes, read, bytes.length - read, read); + if (bytesRead === 0) break; + read += bytesRead; + } + return { name: paths.basename(file), bytes: bytes.subarray(0, read) }; + } finally { + await handle.close(); + } +} + +/** + * Whether `path` may be shown and, once confirmed, opened on `platform`: no character that hides or moves what the + * prompt shows (controls, format characters such as bidirectional and zero-width ones, whitespace but single spaces + * between words: Opus E5), and on Windows no UNC, device or `//host` path and no reserved device name (Opus E9). + */ +function attachable(path: string, platform: NodeJS.Platform): boolean { + if (/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]|[^\S ]| {2}|^ | $/u.test(path)) return false; + if (platform !== "win32") return true; + if (/^[\\/]{2}/.test(path)) return false; + return !path + .split(/[\\/]/) + .some((segment) => /^(?:con|prn|aux|nul|com\d|lpt\d|conin\$|conout\$)(?:\..*)?$/i.test(segment)); +} + +/** The prompt's lines for the file at `file`: its name, its directory, then `question`. */ +function fileLines(file: string, paths: typeof posix, question: string): string[] { + return [`File: ${paths.basename(file)}`, `In: ${paths.dirname(file)}`, question]; +} + +function cannotAttach(): Error { + return new Error("attach: that file cannot be attached"); +} + +/** `text` with control and format characters (bidirectional, zero width) removed: sandboxed text in the prompt. */ +function visible(text: string): string { + return text.replace(/[\p{Cc}\p{Cf}]/gu, ""); +} + +/** `url` when it is an http or https URL without user info; else `undefined`. */ +function openable(url: string): URL | undefined { + try { + const value = new URL(url); + const web = value.protocol === "http:" || value.protocol === "https:"; + return web && value.username === "" && value.password === "" ? value : undefined; + } catch { + return undefined; + } +} + +/** + * The command and arguments opening `url` on `platform`, as the coding agent opens a browser (`open-browser.ts`): never + * a shell (on Windows not `cmd /c start`, whose parsing runs what follows a `&`): `rundll32 + * url.dll,FileProtocolHandler ` on Windows, `open` on macOS, `xdg-open` elsewhere, the URL one argument. + */ +export function openerCommand(url: string, platform: NodeJS.Platform = process.platform): [string, string[]] { + if (platform === "win32") return ["rundll32", ["url.dll,FileProtocolHandler", url]]; + return [platform === "darwin" ? "open" : "xdg-open", [url]]; +} + +/** Opens `url` with the platform's opener (`openerCommand`), detached, best effort: a missing opener is no crash. */ +async function platformOpener(url: string): Promise { + const [command, args] = openerCommand(url); + // The root as its directory: no opener needs the launcher's (a WSL path too long for explorer.exe, pi #11592). + spawn(command, args, { stdio: "ignore", detached: true, cwd: "/", shell: false }) + .on("error", () => {}) + .unref(); +} diff --git a/packages/chord/src/next-node/tui/credentials.ts b/packages/chord/src/next-node/tui/credentials.ts new file mode 100644 index 000000000..35fc0b970 --- /dev/null +++ b/packages/chord/src/next-node/tui/credentials.ts @@ -0,0 +1,71 @@ +// The launcher's credentials (slice 11 §4, decision "The app kit, point 3" (c)): a device token the launcher keeps, +// never the sandbox. It is given, read from a token file (the owner token file on the server's machine, slice 13), or +// got by device pairing: the launcher asks for a code, shows it, and polls until the owner approved it in a browser; +// a paired token may be saved to a file (mode 0600) for the next start. + +import { readFile } from "node:fs/promises"; +import { writeSecret } from "../app/secrets.ts"; + +/** Where the launcher's token comes from. */ +export type TerminalCredentials = + | { readonly kind: "token"; readonly token: string } + | { readonly kind: "file"; readonly path: string } + /** Pairing with the server, the device named `label`; the token saved to `save` (created, mode 0600) when given. */ + | { readonly kind: "pairing"; readonly label: string; readonly save?: string }; + +/** How often a pairing is polled. */ +const POLL_MS = 2000; + +/** The launcher's token for the server at `url`; pairing shows its code through `say`. */ +export async function tokenOf( + credentials: TerminalCredentials, + url: URL, + say: (line: string) => void, + signal: AbortSignal, +): Promise { + switch (credentials.kind) { + case "token": + return credentials.token; + case "file": + return (await readFile(credentials.path, "utf8")).trim(); + case "pairing": + return pair(credentials, url, say, signal); + } +} + +/** Pairs this device: a code shown, the approval polled for, the token returned (and saved when asked). */ +async function pair( + credentials: Extract, + url: URL, + say: (line: string) => void, + signal: AbortSignal, +): Promise { + const asked = await post(new URL("/_chord/pair", url), { label: credentials.label }, signal); + if (!asked.ok) throw new Error(`terminal: the server refused to pair (${asked.status})`); + const { code, poll } = (await asked.json()) as { readonly code?: unknown; readonly poll?: unknown }; + // A trust boundary: the server's answer is shown on the terminal and sent back. + if (typeof code !== "string" || !/^[A-Za-z0-9-]{4,32}$/.test(code) || typeof poll !== "string") { + throw new Error("terminal: the server's pairing answer is malformed"); + } + say(`Pairing code ${code}: approve it at ${new URL(`/_chord/pair/${code}`, url)} (signed in as the owner).`); + for (;;) { + await new Promise((resolve) => setTimeout(resolve, POLL_MS)); + signal.throwIfAborted(); + const polled = await post(new URL("/_chord/pair/poll", url), { poll }, signal); + if (!polled.ok) throw new Error("terminal: the pairing ended without approval"); + const answer = (await polled.json()) as { readonly status: string; readonly token?: string }; + if (answer.status !== "approved" || answer.token === undefined) continue; + if (credentials.save !== undefined) await writeSecret(credentials.save, answer.token, "create"); + return answer.token; + } +} + +/** A JSON post. */ +function post(url: URL, body: unknown, signal: AbortSignal): Promise { + return fetch(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + signal, + }); +} diff --git a/packages/chord/src/next-node/tui/filter.ts b/packages/chord/src/next-node/tui/filter.ts new file mode 100644 index 000000000..be11dcb19 --- /dev/null +++ b/packages/chord/src/next-node/tui/filter.ts @@ -0,0 +1,75 @@ +// The terminal's output filter (slice 11 §4; p2's `tui-filter.ts`): the only path from the sandbox to the real +// terminal. A frame's text and the escape sequences a terminal UI needs to draw pass (cursor moves, erasing, colours, +// the alternate screen, bracketed paste, synchronized output, mouse reporting); every other sequence is dropped, so +// sandboxed code cannot set the clipboard (OSC 52), retitle or reconfigure the terminal, or smuggle a sequence across +// frames: an unterminated one ends with its frame. + +/** The longest frame the sandbox may write. */ +export const MAX_FRAME = 64 * 1024; + +const ESC = "\x1b"; + +/** DEC private modes output may set: cursor visibility, bracketed paste, sync, alternate screen, autowrap, mouse. */ +const MODES: ReadonlySet = new Set(["25", "2004", "2026", "1049", "7", "1000", "1002", "1006", "1004"]); + +/** `output` with only the allowed sequences; throws for a frame over `MAX_FRAME`. */ +export function filterOutput(output: string): string { + if (output.length > MAX_FRAME || Buffer.byteLength(output) > MAX_FRAME) { + throw new RangeError("terminal: a frame over 64 KiB"); + } + let kept = ""; + for (let index = 0; index < output.length; index++) { + const code = output.charCodeAt(index); + if (code === 27 || (code >= 0x80 && code <= 0x9f)) { + const [next, sequence] = escapeAt(output, index); + index = next; + kept += sequence; + continue; + } + if (code === 9 || code === 10 || code === 13 || (code >= 0x20 && code !== 0x7f)) kept += output[index]; + } + return kept; +} + +/** The escape at `index`: the index of its last character, and the sequence kept (`""` when dropped). */ +function escapeAt(output: string, start: number): [number, string] { + let index = start; + const code = output.charCodeAt(index); + const introducer = code === 27 ? (output[++index] ?? "") : output[index]!; + if (introducer === "[" || code === 0x9b) { + const body = index + 1; + while ( + index + 1 < output.length && + output.charCodeAt(index + 1) >= 0x20 && + output.charCodeAt(index + 1) <= 0x3f + ) { + index++; + } + const final = output[++index] ?? ""; + const text = output.slice(body, index); + return [index, code === 27 && final !== "" && allowedCsi(text, final) ? `${ESC}[${text}${final}` : ""]; + } + if ((introducer !== "" && "]PX^_".includes(introducer)) || [0x90, 0x98, 0x9d, 0x9e, 0x9f].includes(code)) { + // A control string (OSC, DCS, ...): dropped up to its terminator, or the end of the frame. + while (++index < output.length) { + if (output.charCodeAt(index) === 0x9c || output.charCodeAt(index) === 7) break; + if (output[index] === ESC && output[index + 1] === "\\") return [index + 1, ""]; + } + return [index, ""]; + } + // Any other ESC sequence with its intermediates (charset selection, ...): dropped. + while (index < output.length && output.charCodeAt(index) >= 0x20 && output.charCodeAt(index) <= 0x2f) index++; + return [index, ""]; +} + +/** Whether the CSI sequence with parameters `body` and final `final` may reach the terminal. */ +function allowedCsi(body: string, final: string): boolean { + if (body.length > 128) return false; + if (final === "h" || final === "l") return body.startsWith("?") && MODES.has(body.slice(1)); + if (final === "m") return /^[\d;:]*$/.test(body) && !/\d{5}/.test(body); + if (!/^[\d;]*$/.test(body) || /\d{5}/.test(body)) return false; + if ("ABCDEFGHfd".includes(final)) return body.split(";").length <= ("Hf".includes(final) ? 2 : 1); + if (final === "J" || final === "K") return /^[012]?$/.test(body); + if (final === "r") return /^(?:\d{1,4};\d{1,4})?$/.test(body); + return (final === "s" || final === "u") && body === ""; +} diff --git a/packages/chord/src/next-node/tui/index.ts b/packages/chord/src/next-node/tui/index.ts new file mode 100644 index 000000000..0d0f52e82 --- /dev/null +++ b/packages/chord/src/next-node/tui/index.ts @@ -0,0 +1,8 @@ +// Chord v2's terminal launcher (`@earendil-works/chord/next/tui/launcher` once v2 has its entry point; slice 11 §4): +// the trusted Node side of a terminal client, running the app's terminal code in the sandbox. + +export type { PromptTerminal } from "./actions.ts"; +export type { TerminalCredentials } from "./credentials.ts"; +export { filterOutput } from "./filter.ts"; +export { type RunningTerminal, startTerminal, type TerminalOptions, terminalFile } from "./launcher.ts"; +export { requireSandboxNode } from "./sandbox.ts"; diff --git a/packages/chord/src/next-node/tui/launcher.ts b/packages/chord/src/next-node/tui/launcher.ts new file mode 100644 index 000000000..66a5b605b --- /dev/null +++ b/packages/chord/src/next-node/tui/launcher.ts @@ -0,0 +1,466 @@ +// `startTerminal` (slice 11 §4, decision "Slice 11, P8"): the trusted launcher of a terminal client. It signs in with +// its token (given, from a file, or by pairing), downloads the terminal kind's base bundle and plugin scripts, and +// runs them in the sandbox, always. It keeps the credentials, the real terminal and the user's choices: the sandbox +// gets input and resizes, its frames pass the output filter, its Chord link is a socket the launcher opens and relays, +// its plugin script fetches go only to the terminal surface's files, and its actions on the machine (copy, open a URL, +// attach a file) are confirmed by the launcher's own prompt. A new base bundle on the server starts a new sandbox. +// Nothing the sandbox sends can end the launcher (fix round 2, Opus E3): its messages are checked (`pipe.ts`), a send +// on a socket not yet open is dropped, every buffer and the requests in flight are bounded, a signal cannot open the +// launcher's inspector, and however the launcher's process ends the terminal is given back (fix round 3, Opus E8, +// E10, E11). During a prompt Ctrl-C ends the client: its keys are the launcher's; and three Ctrl-C within a second +// always end it (Opus R2). + +import type { Readable, Writable } from "node:stream"; +import { WebSocket } from "ws"; +import type { ColorDepth } from "../../next/surfaces/index.ts"; +import type { BridgeRequest, SandboxMessage } from "../../next-tui/bridge.ts"; +import type { TuiStart } from "../surfaces/tui.ts"; +import { act, attach, type PromptTerminal, promptLines } from "./actions.ts"; +import { type TerminalCredentials, tokenOf } from "./credentials.ts"; +import { filterOutput } from "./filter.ts"; +import { MAX_BUNDLE, requireSandboxNode, Sandbox } from "./sandbox.ts"; + +/** What `startTerminal` takes. */ +export type TerminalOptions = { + /** The app's server (`http://127.0.0.1:8788`). */ + readonly url: string; + readonly credentials: TerminalCredentials; + /** The terminal's input and output: the process's by default (raw mode while running, when a TTY). */ + readonly input?: Readable & { setRawMode?(raw: boolean): unknown; readonly isTTY?: boolean }; + readonly output?: Writable & { readonly columns?: number; readonly rows?: number }; + /** How many colours the terminal shows: true colour when `COLORTERM` says so, else 256. */ + readonly colors?: ColorDepth; + /** The terminal's background, choosing between the user's dark and light theme: dark by default. */ + readonly mode?: "dark" | "light"; + /** Opens a URL the user confirmed: the platform's opener by default. */ + readonly opener?: (url: string) => Promise; +}; + +/** A running terminal client: its exit code once it ends, and its stop. */ +export type RunningTerminal = { readonly exited: Promise; stop(): Promise }; + +/** The terminal surface's files (the server's `TUI_PREFIX`). */ +const PREFIX = "/_chord/tui/"; + +/** + * `path` resolved on the server `server` when it names one of the terminal surface's files (dot segments and encodings + * resolved first: Opus M2, Fable L1); throws otherwise: the sandbox reaches nothing else of the server through the + * launcher's token. + */ +export function terminalFile(path: unknown, server: URL): string { + const target = typeof path === "string" ? new URL(path, server) : undefined; + if ( + target === undefined || + target.origin !== server.origin || + target.search !== "" || + !TERMINAL_FILE.test(target.pathname) + ) { + throw new Error("terminal: no terminal file"); + } + return target.pathname; +} + +/** A terminal surface file's path, as the sandbox may ask for it. */ +const TERMINAL_FILE = /^\/_chord\/tui\/(?:plugin\/)?[\w.~@-]+$/; + +/** The most frames held while a prompt is open: 1 MiB (characters); later ones are dropped. */ +const MAX_HELD = 1024 * 1024; + +/** The most link sockets the sandbox may have open at once. */ +const MAX_SOCKETS = 4; + +/** The most bytes a link socket may have waiting to go out; a sandbox sending more loses the socket. */ +const MAX_SOCKET_BUFFER = 16 * 1024 * 1024; + +/** How long after a prompt appears its keys are ignored: a key typed for the sandbox does not answer it (Opus E4). */ +const PROMPT_GUARD_MS = 300; + +/** + * The terminal modes the sandbox may have set, back to the shell's: sync, mouse, focus, paste, cursor, the scroll region + * (the cursor kept where it was) and autowrap (Opus E11), screen, colours. + */ +const RESET_MODES = + "\x1b[?2026l\x1b[?1000l\x1b[?1002l\x1b[?1006l\x1b[?1004l\x1b[?2004l\x1b[?25h\x1b7\x1b[r\x1b8\x1b[?7h\x1b[?1049l\x1b[0m"; + +/** The most requests of one sandbox the launcher answers at once; more are refused at once (Opus E10). */ +const MAX_REQUESTS = 8; + +/** + * Ctrl-C: during a prompt it ends the client, so a sandbox asking again and again can always be left (Opus E11); and + * three within a second always end it, whatever the sandbox does with them (Opus R2). + */ +const INTERRUPT = "\x03"; + +/** How many Ctrl-C within `INTERRUPT_WINDOW_MS` end the client however the sandbox takes them. */ +const INTERRUPTS_TO_END = 3; +const INTERRUPT_WINDOW_MS = 1000; + +/** Starts a terminal client; rejects when it cannot sign in or download, or Node is too old for the sandbox. */ +export async function startTerminal(options: TerminalOptions): Promise { + requireSandboxNode(); + const output = options.output ?? process.stdout; + const url = new URL(options.url); + const abort = new AbortController(); + // What the launcher prints of the server's (the pairing's code) passes the output filter too. + const token = await tokenOf( + options.credentials, + url, + (line) => output.write(`${filterOutput(line)}\r\n`), + abort.signal, + ); + const launcher = new Launcher(url, token, options); + await launcher.start(); + return { exited: launcher.exited, stop: () => launcher.finish(0) }; +} + +/** One terminal client's launcher. */ +class Launcher { + readonly #url: URL; + readonly #headers: Readonly>; + readonly #options: TerminalOptions; + readonly #input: NonNullable; + readonly #output: NonNullable; + readonly #sockets = new Map(); + #sandbox: Sandbox | undefined; + /** The running base bundle's hash: an `outdated` sandbox restarts only for another. */ + #base = ""; + /** + * Whether an action is being confirmed (one at a time), from its request to its end: meanwhile the sandbox's frames + * are held (`#held`, at most `MAX_HELD` bytes), so nothing draws over the prompt, even before it appears. + */ + #confirming = false; + /** The running sandbox's requests being answered (at most `MAX_REQUESTS`). */ + #requests = 0; + /** The prompt open: its lines, its answer, and when its keys start to count. Keys go to it, not the sandbox. */ + #prompt: + | { readonly lines: readonly string[]; readonly answer: (yes: boolean) => void; readonly ready: number } + | undefined; + #held: string[] = []; + #heldBytes = 0; + #finished = false; + #exit!: (code: number) => void; + readonly exited = new Promise((resolve) => { + this.#exit = resolve; + }); + readonly #onInput = (data: Buffer | string): void => { + const text = data.toString(); + if (this.#interrupted(text)) { + void this.finish(130); + return; + } + if (this.#prompt === undefined) this.#sandbox?.post({ kind: "input", data: text }); + else if (text.includes(INTERRUPT)) void this.finish(130); + else if (Date.now() >= this.#prompt.ready) this.#answered(text === "y" || text === "Y"); + }; + readonly #onResize = (): void => { + if (this.#prompt !== undefined) this.#drawPrompt(); + this.#sandbox?.post({ kind: "resize", columns: this.#columns, rows: this.#rows }); + }; + /** When the last Ctrl-C keys came: three within a second end the client. */ + #interrupts: number[] = []; + /** The terminal given back if the launcher's process ends without `finish` (a crash). */ + readonly #onExit = (): void => { + this.#output.write(RESET_MODES); + if (this.#input.isTTY === true) this.#input.setRawMode?.(false); + }; + + constructor(url: URL, token: string, options: TerminalOptions) { + this.#url = url; + this.#headers = { authorization: `Bearer ${token}` }; + this.#options = options; + this.#input = options.input ?? process.stdin; + this.#output = options.output ?? process.stdout; + } + + /** Whether `text` brings the third Ctrl-C within a second (each still goes to the sandbox otherwise). */ + #interrupted(text: string): boolean { + const now = Date.now(); + for (const character of text) if (character === INTERRUPT) this.#interrupts.push(now); + this.#interrupts = this.#interrupts.filter((at) => now - at < INTERRUPT_WINDOW_MS); + return this.#interrupts.length >= INTERRUPTS_TO_END; + } + + // A pseudo-terminal may report 0 (Opus E4). + get #columns(): number { + return this.#output.columns || 80; + } + + get #rows(): number { + return this.#output.rows || 24; + } + + /** Downloads the base bundle and the plugins, starts the sandbox and takes the terminal. */ + async start(): Promise { + await this.#startSandbox(); + if (this.#input.isTTY === true) this.#input.setRawMode?.(true); + this.#input.on("data", this.#onInput); + this.#output.on("resize", this.#onResize); + process.once("exit", this.#onExit); + // Node opens its inspector on SIGUSR1 unless the signal has a listener: a sandbox may signal its launcher, and + // an inspector would give any local process the launcher's token and files (Opus E8). Windows has no SIGUSR1. + if (process.platform !== "win32") process.on("SIGUSR1", ignoreSignal); + } + + /** Ends the client with `code`: the terminal given back, the sockets closed, the sandbox ended. */ + async finish(code: number): Promise { + if (this.#finished) return; + this.#finished = true; + this.#input.off("data", this.#onInput); + this.#output.off("resize", this.#onResize); + process.off("exit", this.#onExit); + if (process.platform !== "win32") process.off("SIGUSR1", ignoreSignal); + if (this.#input.isTTY === true) this.#input.setRawMode?.(false); + this.#prompt?.answer(false); + this.#prompt = undefined; + this.#output.write(RESET_MODES); + this.#closeSockets(); + await this.#sandbox?.terminate(); + this.#exit(code); + } + + /** A new sandbox with what the server serves now. */ + async #startSandbox(): Promise { + const start = JSON.parse(await this.#get(`${PREFIX}start`)) as TuiStart; + const [source, combined] = await Promise.all([this.#get(start.script), this.#get(start.combined)]); + this.#base = start.base; + const colors = + this.#options.colors ?? (/truecolor|24bit/.test(process.env.COLORTERM ?? "") ? "truecolor" : "256"); + const bridge = { + base: start.base, + combined, + columns: this.#columns, + rows: this.#rows, + colors, + mode: this.#options.mode ?? "dark", + } as const; + this.#requests = 0; + this.#sandbox = await Sandbox.start( + source, + bridge, + (message) => this.#message(message), + () => void this.finish(1), + ); + } + + /** One message of the sandbox; one the launcher fails on ends the client, never the launcher's process. */ + #message(message: SandboxMessage): void { + try { + this.#handle(message); + } catch { + void this.finish(1); + } + } + + #handle(message: SandboxMessage): void { + switch (message.kind) { + case "write": + this.#write(message.data); + return; + case "socket_open": + this.#openSocket(message.socket); + return; + case "socket_send": + this.#send(message.socket, message.bytes); + return; + case "socket_close": + this.#sockets.get(message.socket)?.close(); + return; + case "request": + void this.#answer(message.id, message.request); + return; + case "outdated": + void this.#outdated(); + return; + case "failed": + this.#output.write("terminal: the terminal client failed to start\n"); + void this.finish(1); + return; + case "exit": + void this.finish(message.code); + } + } + + /** Writes the sandbox's frame through the output filter (held while a prompt is open); one over 64 KiB is dropped. */ + #write(data: string): void { + if (this.#confirming) { + if (typeof data === "string" && this.#heldBytes + data.length <= MAX_HELD) { + this.#held.push(data); + this.#heldBytes += data.length; + } + return; + } + try { + this.#output.write(filterOutput(data)); + } catch { + // Over 64 KiB (or no text): dropped, as an escape it could hide would be. + } + } + + /** Opens socket `socket` to the links endpoint with the launcher's token, relaying it to the sandbox. */ + #openSocket(socket: number): void { + // A few at a time: the connection loop needs one; a runaway sandbox gets no more. + if (this.#sockets.size >= MAX_SOCKETS) { + this.#sandbox?.post({ kind: "socket_closed", socket }); + return; + } + const target = new URL("/chord", this.#url); + target.protocol = target.protocol === "https:" ? "wss:" : "ws:"; + const ws = new WebSocket(target, { headers: this.#headers }); + this.#sockets.set(socket, ws); + ws.on("open", () => this.#sandbox?.post({ kind: "socket_opened", socket })); + ws.on("message", (data: Buffer) => { + this.#sandbox?.post({ kind: "socket_message", socket, bytes: new Uint8Array(data) }); + }); + const closed = (): void => { + if (this.#sockets.get(socket) !== ws) return; + this.#sockets.delete(socket); + this.#sandbox?.post({ kind: "socket_closed", socket }); + }; + ws.on("close", closed); + ws.on("error", closed); + } + + /** + * Sends `bytes` on socket `socket` once it is open (before, the sandbox's link holds its messages; a send then is + * dropped: Opus E3); a socket with more than 16 MiB waiting is closed. + */ + #send(socket: number, bytes: Uint8Array): void { + const ws = this.#sockets.get(socket); + if (ws?.readyState !== WebSocket.OPEN) return; + if (ws.bufferedAmount + bytes.length > MAX_SOCKET_BUFFER) ws.terminate(); + else ws.send(bytes); + } + + /** Answers the sandbox's request `id`. */ + async #answer(id: number, request: BridgeRequest): Promise { + // The sandbox that asked: a reply never goes to the one a restart started meanwhile. + const sandbox = this.#sandbox; + if (sandbox === undefined) return; + if (this.#requests >= MAX_REQUESTS) { + sandbox.post({ kind: "refused", id, message: "terminal: too many requests at once" }); + return; + } + this.#requests++; + try { + let value: unknown; + if (request.kind === "fetch") value = await this.#get(terminalFile(request.path, this.#url)); + else if (request.kind === "base") value = JSON.parse(await this.#get(`${PREFIX}base`)); + else if (request.kind === "attach") { + const file = await this.#confirm(() => attach(request.path, this.#prompter())); + if (file !== null) { + sandbox.post({ kind: "attached", id, name: file.name, bytes: file.bytes }); + return; + } + value = null; + } else value = await this.#confirm(() => act(request, this.#prompter(), this.#options.opener)); + sandbox.post({ kind: "reply", id, value }); + } catch (error) { + sandbox.post({ kind: "refused", id, message: error instanceof Error ? error.message : String(error) }); + } finally { + if (this.#sandbox === sandbox) this.#requests--; + } + } + + /** + * Runs the action `run` (it asks the user), one at a time: a second is refused while one runs. Afterwards the held + * frames are written and the sandbox asked to redraw. + */ + async #confirm(run: () => Promise): Promise { + if (this.#confirming) throw new Error("terminal: a prompt is open"); + this.#confirming = true; + try { + return await run(); + } finally { + this.#confirming = false; + const held = this.#held; + this.#held = []; + this.#heldBytes = 0; + for (const frame of held) this.#write(frame); + this.#sandbox?.post({ kind: "resize", columns: this.#columns, rows: this.#rows }); + } + } + + /** The launcher's own prompt on the real terminal, for the action being confirmed: its key not the sandbox's. */ + #prompter(): PromptTerminal { + return { + columns: this.#columns, + ask: (lines) => { + // Shown whole or not at all: a prompt is never cut. + if (promptLines(lines, this.#columns).length >= this.#rows) { + return Promise.reject(new Error("terminal: the prompt does not fit the terminal")); + } + return new Promise((answer) => { + this.#prompt = { lines, answer, ready: Date.now() + PROMPT_GUARD_MS }; + this.#drawPrompt(); + }); + }, + write: (data) => this.#output.write(data), + }; + } + + /** + * Draws the open prompt on the bottom lines after closing a synchronized update the sandbox began: its attributes + * reset and its colours explicit, black on white, so no state the sandbox left (concealed, black on black: Opus E4) + * applies to it. + */ + #drawPrompt(): void { + const lines = promptLines(this.#prompt!.lines, this.#columns); + const top = Math.max(1, this.#rows - lines.length + 1); + const drawn = lines.map((line, index) => `\x1b[${top + index};1H\x1b[0;30;47m\x1b[2K ${line} `); + this.#output.write(`\x1b[?2026l\x1b[s${drawn.join("")}\x1b[0m`); + } + + /** The prompt answered: cleared, the cursor back where the sandbox had it. */ + #answered(yes: boolean): void { + const { lines, answer } = this.#prompt!; + this.#prompt = undefined; + const rows = promptLines(lines, this.#columns); + const top = Math.max(1, this.#rows - rows.length + 1); + const cleared = rows.map((_row, index) => `\x1b[${top + index};1H\x1b[0m\x1b[2K`).join(""); + this.#output.write(`${cleared}\x1b[u`); + answer(yes); + } + + /** The sandbox says its code is outdated: a new sandbox only when the server's base bundle is another. */ + async #outdated(): Promise { + const base = JSON.parse(await this.#get(`${PREFIX}base`).catch(() => "null")) as unknown; + if (typeof base === "string" && base !== this.#base) await this.#restart(); + } + + /** The server has a new base bundle: a new sandbox with it, the old one ended. */ + async #restart(): Promise { + this.#closeSockets(); + await this.#sandbox?.terminate(); + this.#sandbox = undefined; + try { + await this.#startSandbox(); + this.#onResize(); + } catch { + await this.finish(1); + } + } + + #closeSockets(): void { + for (const ws of this.#sockets.values()) ws.close(); + this.#sockets.clear(); + } + + /** The text at `path` on the server, signed in; at most 8 MiB, read no further. */ + async #get(path: string): Promise { + const response = await fetch(new URL(path, this.#url), { headers: this.#headers }); + if (!response.ok || response.body === null) throw new Error(`terminal: GET ${path} answered ${response.status}`); + if (Number(response.headers.get("content-length") ?? 0) > MAX_BUNDLE) + throw new Error(`terminal: ${path} exceeds 8 MiB`); + const chunks: Uint8Array[] = []; + let size = 0; + for await (const chunk of response.body) { + size += chunk.length; + if (size > MAX_BUNDLE) throw new Error(`terminal: ${path} exceeds 8 MiB`); + chunks.push(chunk); + } + return new TextDecoder().decode(Buffer.concat(chunks)); + } +} + +/** The launcher's SIGUSR1 listener: the signal does nothing. */ +function ignoreSignal(): void {} diff --git a/packages/chord/src/next-node/tui/pipe.ts b/packages/chord/src/next-node/tui/pipe.ts new file mode 100644 index 000000000..a775303a5 --- /dev/null +++ b/packages/chord/src/next-node/tui/pipe.ts @@ -0,0 +1,132 @@ +// The launcher's end of the sandbox's pipes (slice 11 §4; fix round 2: the sandbox a child process): messages as +// plain length-prefixed binary frames, not Node's IPC. A frame is a 4-byte big-endian length, then the payload: a +// 4-byte big-endian header length, the header (the message as JSON, its body field left out), and the body (the raw +// bytes of `bytes`, or the UTF-8 text of `data`). The sandbox's bootstrap has the same codec (`sandbox.ts`). What the +// sandbox sends is untrusted: each frame is bounded, and each message checked here before the launcher acts on it. + +import type { BridgeRequest, LauncherMessage, SandboxMessage } from "../../next-tui/bridge.ts"; + +/** The largest frame either side takes: 17 MiB (an attached file of 16 MiB and its header). */ +export const MAX_PIPE_FRAME = 17 * 1024 * 1024; + +/** The largest header of a frame from the sandbox: 1 MiB (a request's text or URL; frames carry their text as body). */ +const MAX_HEADER = 1024 * 1024; + +/** `message` as one frame for the sandbox. */ +export function launcherFrame(message: LauncherMessage): Buffer { + if (message.kind === "input") return frame({ kind: "input" }, Buffer.from(message.data)); + if (message.kind === "socket_message") return frame({ kind: message.kind, socket: message.socket }, message.bytes); + if (message.kind === "attached") + return frame({ kind: message.kind, id: message.id, name: message.name }, message.bytes); + return frame(message, undefined); +} + +/** The first frame: the bridge's start and the base bundle's text as body. */ +export function startFrame(start: unknown, source: string): Buffer { + return frame({ kind: "start", start }, Buffer.from(source)); +} + +/** A frame of `header` and `body`. */ +function frame(header: object, body: Uint8Array | undefined): Buffer { + const json = Buffer.from(JSON.stringify(header)); + const prefix = Buffer.allocUnsafe(8); + prefix.writeUInt32BE(4 + json.length + (body?.length ?? 0), 0); + prefix.writeUInt32BE(json.length, 4); + return Buffer.concat(body === undefined ? [prefix, json] : [prefix, json, body]); +} + +/** + * Cuts the sandbox's output into messages as chunks arrive; throws (the sandbox is ended) on a frame over + * `MAX_PIPE_FRAME`, a header over 1 MiB, or a message of no known shape. Each byte is copied in once, however small the + * chunks the sandbox writes. + */ +export class FrameReader { + #buffer = Buffer.alloc(64 * 1024); + #start = 0; + #end = 0; + + /** The messages complete with `chunk`. */ + push(chunk: Buffer): SandboxMessage[] { + this.#append(chunk); + const messages: SandboxMessage[] = []; + while (this.#end - this.#start >= 4) { + const length = this.#buffer.readUInt32BE(this.#start); + if (length < 4 || length > MAX_PIPE_FRAME) throw new Error("terminal: the sandbox sent a malformed frame"); + if (this.#end - this.#start < 4 + length) break; + // The message copies what it keeps: the buffer is reused. + messages.push(sandboxMessage(this.#buffer.subarray(this.#start + 4, this.#start + 4 + length))); + this.#start += 4 + length; + } + if (this.#start === this.#end) this.#start = this.#end = 0; + return messages; + } + + /** Copies `chunk` in after the unread bytes, moving them to the front or growing the buffer when it is full. */ + #append(chunk: Buffer): void { + const unread = this.#end - this.#start; + if (this.#end + chunk.length > this.#buffer.length) { + const needed = unread + chunk.length; + const target = + needed <= this.#buffer.length + ? this.#buffer + : Buffer.alloc(Math.max(needed, Math.min(this.#buffer.length * 2, MAX_PIPE_FRAME + 4))); + this.#buffer.copy(target, 0, this.#start, this.#end); + this.#buffer = target; + this.#start = 0; + this.#end = unread; + } + chunk.copy(this.#buffer, this.#end); + this.#end += chunk.length; + } +} + +/** + * The message in `payload` (a frame from the sandbox, without its length); throws for anything else. Trust boundary: + * the sandbox's pipe. + */ +function sandboxMessage(payload: Buffer): SandboxMessage { + const headerLength = payload.readUInt32BE(0); + if (headerLength > MAX_HEADER || 4 + headerLength > payload.length) throw malformed(); + const header = JSON.parse(payload.toString("utf8", 4, 4 + headerLength)) as Record | null; + const body = payload.subarray(4 + headerLength); + if (typeof header !== "object" || header === null) throw malformed(); + const socket = header.socket; + switch (header.kind) { + case "write": + return { kind: "write", data: body.toString("utf8") }; + case "socket_send": + if (!Number.isSafeInteger(socket)) throw malformed(); + // A copy: the payload's memory is the reader's. + return { kind: "socket_send", socket: socket as number, bytes: new Uint8Array(body) }; + case "socket_open": + case "socket_close": + if (!Number.isSafeInteger(socket)) throw malformed(); + return { kind: header.kind, socket: socket as number }; + case "request": + if (!Number.isSafeInteger(header.id)) throw malformed(); + return { kind: "request", id: header.id as number, request: bridgeRequest(header.request) }; + case "outdated": + return { kind: "outdated" }; + case "failed": + return { kind: "failed", message: String(header.message) }; + case "exit": + return { kind: "exit", code: Number.isSafeInteger(header.code) ? (header.code as number) : 1 }; + default: + throw malformed(); + } +} + +/** The request `value` names; throws for anything else. */ +function bridgeRequest(value: unknown): BridgeRequest { + const request = (typeof value === "object" && value !== null ? value : {}) as Record; + if (request.kind === "base") return { kind: "base" }; + if (request.kind === "fetch" && typeof request.path === "string") return { kind: "fetch", path: request.path }; + if (request.kind === "copy" && typeof request.text === "string") return { kind: "copy", text: request.text }; + if (request.kind === "open" && typeof request.url === "string") return { kind: "open", url: request.url }; + if (request.kind === "attach" && typeof request.path === "string") return { kind: "attach", path: request.path }; + throw malformed(); +} + +function malformed(): Error { + return new Error("terminal: the sandbox sent a malformed message"); +} diff --git a/packages/chord/src/next-node/tui/sandbox.ts b/packages/chord/src/next-node/tui/sandbox.ts new file mode 100644 index 000000000..c9b6384d9 --- /dev/null +++ b/packages/chord/src/next-node/tui/sandbox.ts @@ -0,0 +1,450 @@ +// The terminal's sandbox (slice 11 §4, decision "Slice 11, P8"; fix round 2: a child process, as the owner decided +// after the worker thread was escaped through its shared realm): a Node child process of the launcher's binary. +// - The operating system's side: its stdin, stdout and stderr are pipes to the launcher (never the terminal: no +// descriptor of the child is a TTY), its environment empty, its directory a private one holding only the bootstrap; +// Node's permission model with nothing allowed but reading the bootstrap (no other file read, no write, no network, +// no child processes, no workers, no addons, no inspector, no WASI) and `node:sqlite` removed +// (`--no-experimental-sqlite`: SQLite opens files past the permission model). An escape from the JavaScript layer +// reaches no terminal, no file, no socket, no launcher memory or token. +// - The JavaScript side, a second layer: downloaded code (the base bundle, a CommonJS script; the plugins' registered +// scripts) is only evaluated by the bootstrap (`vm.Script` with no dynamic import callback, so `import()` throws), +// with a `require` answering the allowed builtins only (`SANDBOX_BUILTINS`); `process` is a substitute with +// `SANDBOX_PROCESS` only. The bootstrap keeps its own references to what it uses before any downloaded code runs. +// Messages go both ways as length-prefixed binary frames (`pipe.ts`). Known gaps, each needing the JavaScript layer +// broken first unless noted: the child may signal processes of the same user (Node has no permission for signals; the +// launcher ignores SIGUSR1, which would open its inspector), read machine details (`os`), learn whether any path exists +// (`require.resolve` fails one way for a file that exists, another for one that does not: Opus E13), and allocate +// memory outside its heap (no limit without a shell's `ulimit`; only the child dies). Needs Node 26.5.1 or later. + +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { BridgeStart, LauncherMessage, SandboxMessage } from "../../next-tui/bridge.ts"; +import { FrameReader, launcherFrame, MAX_PIPE_FRAME, startFrame } from "./pipe.ts"; + +/** The largest base bundle a sandbox runs: 8 MiB. */ +export const MAX_BUNDLE = 8 * 1024 * 1024; + +/** + * The builtins code in the sandbox may `require`, each computation in memory without I/O, descriptors or machine + * details: bytes and text (`buffer`, `string_decoder`, `punycode`, `querystring`), events and streams in memory + * (`events`, `stream` and its submodules), formatting and type checks (`util`, `util/types`), URL and path strings + * (`url`, `path`), timers, assertions, hashing and randomness (`crypto`: no keys from files, no network) and compression + * in memory (`zlib`). Not `fs`, `tty`, `net`, `dgram`, `dns`, `http`, `https`, `http2`, `tls`, `child_process`, + * `cluster`, `worker_threads`, `sqlite`, `inspector`, `vm`, `v8`, `os`, `module`, `process`, `perf_hooks`, `wasi`, + * `repl`, `readline`, `trace_events` or anything else. + */ +export const SANDBOX_BUILTINS: readonly string[] = [ + "assert", + "assert/strict", + "buffer", + "crypto", + "events", + "path", + "path/posix", + "path/win32", + "punycode", + "querystring", + "stream", + "stream/consumers", + "stream/promises", + "stream/web", + "string_decoder", + "timers", + "timers/promises", + "url", + "util", + "util/types", + "zlib", +]; + +/** + * The sandbox's `process`: a frozen plain object in place of Node's (no event emitter, so no prototype shared with + * Node's internals leads back to the real one). Its properties, each harmless: which Node it is (`arch`, `platform`, + * `version`, `versions`, `release`), time (`nextTick`, `hrtime`, `uptime`), ending the sandbox (`exit`), the permission + * checks (`permission`), the allowed builtins (`getBuiltinModule`), fixed values (`env` empty, `argv`, `argv0`, + * `execArgv`, `execPath`, `title`, `cwd` `/`), `stdout` (to the terminal through the bridge) and `stderr` (dropped), + * and listener calls that keep nothing (`on`, `once`, `off`, `addListener`, `removeListener`, `removeAllListeners`, + * `emit`, `emitWarning`, `listenerCount`, `listeners`). Not `binding`, `_linkedBinding`, `dlopen`, `_rawDebug`, `kill`, + * `execve`, `chdir`, `abort`, `reallyExit`, `stdin`, `openStdin`, `report`, `pid`, `ppid`, `config`, the user and group + * calls, `loadEnvFile`, `moduleLoadList`, resource and handle listings or any other. + */ +export const SANDBOX_PROCESS: readonly string[] = [ + "addListener", + "arch", + "argv", + "argv0", + "cwd", + "emit", + "emitWarning", + "env", + "execArgv", + "execPath", + "exit", + "getBuiltinModule", + "hrtime", + "listenerCount", + "listeners", + "nextTick", + "off", + "on", + "once", + "permission", + "platform", + "release", + "removeAllListeners", + "removeListener", + "stderr", + "stdout", + "title", + "uptime", + "version", + "versions", +]; + +/** + * The child's Node flags for its script `script`: the permission model allowing only reading that file, no SQLite, a + * 96 MiB heap. Exported for the tests of the operating system's layer alone. + */ +export function sandboxFlags(script: string): string[] { + return ["--permission", `--allow-fs-read=${script}`, "--no-experimental-sqlite", "--max-old-space-size=96", script]; +} + +/** The bootstrap's file in the sandbox's directory: a script, not `-e`, which puts every builtin on `globalThis`. */ +export const BOOTSTRAP_FILE = "bootstrap.cjs"; + +/** The most bytes waiting for the sandbox to read them before it counts as stuck: 64 MiB. */ +const MAX_UNREAD = 64 * 1024 * 1024; + +/** Throws unless this Node is 26.5.1 or later, the Node the sandbox's permissions were checked on. */ +export function requireSandboxNode(version: string = process.versions.node): void { + const [major = 0, minor = 0, patch = 0] = version.split(".").map(Number); + if (major > 26 || (major === 26 && (minor > 5 || (minor === 5 && patch >= 1)))) return; + throw new Error(`the terminal client needs Node 26.5.1 or later for its sandbox (this is Node ${version})`); +} + +/** + * The child's environment: empty, but for `LD_LIBRARY_PATH` when the launcher has one: the launcher needed it to run + * this same binary (a Node built against libraries outside the system's paths), and the server cannot influence it. + */ +export function sandboxEnvironment(): Record { + const libraries = process.env.LD_LIBRARY_PATH; + return libraries === undefined ? {} : { LD_LIBRARY_PATH: libraries }; +} + +/** A running sandbox: messages both ways, and its end. */ +export class Sandbox { + readonly #child: ChildProcessWithoutNullStreams; + readonly #directory: string; + readonly #onMessage: (message: SandboxMessage) => void; + readonly #onEnd: (reason: string) => void; + readonly #reader = new FrameReader(); + readonly #exited: Promise; + #ended = false; + /** Whether the child has sent anything: its bootstrap was read, so the file goes. */ + #running = false; + + /** + * Starts a sandbox running the base bundle `source` (a CommonJS script) with `start` on its bridge; `onMessage` gets + * what it sends, `onEnd` its end (an exit, a crash, its memory limit, a malformed frame), once. + */ + static async start( + source: string, + start: BridgeStart, + onMessage: (message: SandboxMessage) => void, + onEnd: (reason: string) => void, + ): Promise { + requireSandboxNode(); + if (Buffer.byteLength(source) > MAX_BUNDLE) throw new Error("terminal: the base bundle exceeds 8 MiB"); + const directory = await mkdtemp(join(tmpdir(), "chord-tui-")); + await writeFile(join(directory, BOOTSTRAP_FILE), BOOTSTRAP); + const sandbox = new Sandbox(directory, onMessage, onEnd); + sandbox.#write(startFrame(start, source)); + return sandbox; + } + + private constructor( + directory: string, + onMessage: (message: SandboxMessage) => void, + onEnd: (reason: string) => void, + ) { + this.#directory = directory; + this.#onMessage = onMessage; + this.#onEnd = onEnd; + this.#child = spawn(process.execPath, sandboxFlags(join(directory, BOOTSTRAP_FILE)), { + stdio: ["pipe", "pipe", "pipe"], + env: sandboxEnvironment(), + cwd: directory, + windowsHide: true, + }); + this.#exited = new Promise((resolve) => this.#child.once("close", () => resolve())); + this.#child.stdout.on("data", (chunk: Buffer) => this.#read(chunk)); + // Dropped: the sandbox writes only through the bridge, and the terminal is the launcher's. + this.#child.stderr.resume(); + // A write after the child ended fails here; its end is reported by `close`. + this.#child.stdin.on("error", () => {}); + this.#child.on("error", (error) => this.#end(`the terminal client failed: ${error.message}`)); + this.#child.on("close", () => { + this.#end("the terminal client exited"); + void rm(this.#directory, { recursive: true, force: true }); + }); + } + + /** Sends `message` to the sandbox; a sandbox not reading what it was sent is ended. */ + post(message: LauncherMessage): void { + if (!this.#ended) this.#write(launcherFrame(message)); + } + + /** Ends the sandbox and waits for its exit. */ + async terminate(): Promise { + this.#ended = true; + this.#child.kill("SIGKILL"); + await this.#exited; + } + + #write(frame: Buffer): void { + if (this.#child.stdin.writableLength + frame.length > MAX_UNREAD + MAX_PIPE_FRAME) { + this.#fail("the terminal client stopped reading"); + return; + } + this.#child.stdin.write(frame); + } + + /** The sandbox's output: its messages, each passed on; a malformed one ends the sandbox. */ + #read(chunk: Buffer): void { + if (this.#ended) return; + if (!this.#running) { + this.#running = true; + // Read once at the start; a launcher killed later leaves at most an empty directory. + void rm(join(this.#directory, BOOTSTRAP_FILE), { force: true }); + } + let messages: SandboxMessage[]; + try { + messages = this.#reader.push(chunk); + } catch (error) { + this.#fail((error as Error).message); + return; + } + for (const message of messages) if (!this.#ended) this.#onMessage(message); + } + + #fail(reason: string): void { + this.#child.kill("SIGKILL"); + this.#end(reason); + } + + #end(reason: string): void { + if (this.#ended) return; + this.#ended = true; + this.#onEnd(reason); + } +} + +/** + * The trusted bootstrap (strict CommonJS, the child's main script). Before any downloaded code runs it keeps its own + * references to what it uses (so code that changes shared prototypes later cannot steer it), closes the file system's + * functions, shadows the real `process`'s listener calls, puts the substitute `process` in place, removes the module + * globals and leaves the bridge on `globalThis`; then it waits for the launcher's start frame and evaluates the base + * bundle through its own `require`. Frames as in `pipe.ts`. + */ +const BOOTSTRAP = `"use strict"; +const real = process; +const getBuiltin = real.getBuiltinModule; +const apply = Reflect.apply; +const uncurry = Function.prototype.call.bind(Function.prototype.bind, Function.prototype.call); +const parse = JSON.parse; +const stringify = JSON.stringify; +const freeze = Object.freeze; +const define = Object.defineProperty; +const BufferFrom = Buffer.from; +const BufferAlloc = Buffer.allocUnsafe; +const BufferConcat = Buffer.concat; +const readUInt32BE = uncurry(Buffer.prototype.readUInt32BE); +const writeUInt32BE = uncurry(Buffer.prototype.writeUInt32BE); +const bufferText = uncurry(Buffer.prototype.toString); +const subarray = uncurry(Buffer.prototype.subarray); +const copyBytes = uncurry(Buffer.prototype.copy); +const isUint8Array = (value) => value instanceof Uint8Array; +const builtin0 = (id) => apply(getBuiltin, real, [id]); +const Script = builtin0("node:vm").Script; +const runInThisContext = uncurry(Script.prototype.runInThisContext); +const nextTick = real.nextTick; +const hrtime = real.hrtime; +const hrtimeBigint = real.hrtime.bigint; +const uptime = real.uptime; +const exit = real.exit; +const input = real.stdin; +const output = real.stdout; +const writeOutput = uncurry(output.write); +const onInput = uncurry(input.on); +const MAX_FRAME = ${MAX_PIPE_FRAME}; +const deny = () => { + throw new Error("unavailable in the terminal's sandbox"); +}; + +// The file system's functions closed: the permission model guards paths, not the pipes' descriptors. +for (const facade of [builtin0("node:fs"), builtin0("node:fs/promises")]) { + for (const name of Object.keys(facade)) { + if (typeof facade[name] === "function") define(facade, name, { value: deny, writable: false, configurable: false, enumerable: true }); + } +} +const v8 = builtin0("node:v8"); +for (const name of ["writeHeapSnapshot", "getHeapSnapshot", "queryObjects", "setFlagsFromString", "takeCoverage", "startCpuProfile", "startHeapProfile"]) { + if (typeof v8[name] === "function") v8[name] = deny; +} +builtin0("node:module").syncBuiltinESMExports(); +// Node emits on its own process (warnings among them): its listener calls are its own properties, not the shared +// prototype's, which downloaded code may change. +const emitter = builtin0("node:events").EventEmitter.prototype; +for (const name of Object.getOwnPropertyNames(emitter)) { + if (name !== "constructor" && typeof emitter[name] === "function") define(real, name, { value: emitter[name], writable: false, configurable: false }); +} +// Node reads properties its process does not have (noDeprecation, throwDeprecation, traceDeprecation, +// traceProcessWarnings, channel, connected, …): such a read goes up the prototype chain, where a getter would get the +// real process as its this (Opus E7: Buffer(1) warns, and the warning reads process.noDeprecation). Every prototype of +// the chain, Object.prototype included, takes no new property and keeps its own ones, so no getter and no other +// prototype gets in. Not frozen: Node assigns over prototype fields (\`this._events ??= {}\`). +for (let prototype = Object.getPrototypeOf(real); prototype !== null; prototype = Object.getPrototypeOf(prototype)) { + for (const key of Reflect.ownKeys(prototype)) define(prototype, key, { configurable: false }); + Object.preventExtensions(prototype); +} + +// The allowed builtins, by name with or without "node:"; looked up in an object of no prototype. +const allowed = Object.create(null); +for (const name of ${JSON.stringify(SANDBOX_BUILTINS)}) allowed[name] = true; +const builtin = (id) => { + if (typeof id !== "string") throw new Error("unavailable in the terminal's sandbox"); + const name = id.length > 5 && id[0] === "n" && id[1] === "o" && id[2] === "d" && id[3] === "e" && id[4] === ":" ? id.slice(5) : id; + if (typeof name !== "string" || allowed[name] !== true) throw new Error(id + ": unavailable in the terminal's sandbox"); + return builtin0("node:" + name); +}; + +// Frames out: the message's header as JSON, its write text or socket bytes as the body. +const send = (message) => { + const kind = message.kind; + let header; + let body; + if (kind === "write") { + const data = String(message.data); + // Refused before it is copied: a huge string costs no buffer of its size. + if (data.length > MAX_FRAME) throw new RangeError("the terminal's bridge takes frames of at most 17 MiB"); + header = { kind }; + body = BufferFrom(data); + } else if (kind === "socket_send") { + if (!isUint8Array(message.bytes)) throw new TypeError("socket_send: bytes expected"); + header = { kind, socket: message.socket }; + body = message.bytes; + } else { + header = message; + } + const json = BufferFrom(stringify(header)); + const length = 4 + json.length + (body === undefined ? 0 : body.length); + if (length > MAX_FRAME) throw new RangeError("the terminal's bridge takes frames of at most 17 MiB"); + const prefix = BufferAlloc(8); + writeUInt32BE(prefix, length, 0); + writeUInt32BE(prefix, json.length, 4); + writeOutput(output, BufferConcat(body === undefined ? [prefix, json] : [prefix, json, body])); +}; + +// The substitute process. +const stdout = freeze({ isTTY: false, write: (data) => (send({ kind: "write", data: String(data) }), true) }); +const listening = (name) => [name, function () { return fake; }]; +const fake = freeze({ + arch: real.arch, + platform: real.platform, + version: real.version, + versions: freeze({ ...real.versions }), + release: freeze({ ...real.release }), + nextTick: (...args) => apply(nextTick, real, args), + hrtime: Object.assign((...args) => apply(hrtime, real, args), { bigint: () => apply(hrtimeBigint, real, []) }), + uptime: () => apply(uptime, real, []), + exit: (code) => apply(exit, real, [code]), + permission: real.permission, + env: freeze({}), + argv: freeze(["node"]), + argv0: "node", + execArgv: freeze([]), + execPath: "node", + title: "node", + cwd: () => "/", + getBuiltinModule: builtin, + stdout, + stderr: freeze({ isTTY: false, write: () => true }), + ...Object.fromEntries(["on", "once", "off", "addListener", "removeListener", "removeAllListeners"].map(listening)), + emit: () => false, + emitWarning: () => {}, + listenerCount: () => 0, + listeners: () => [], +}); +define(globalThis, "process", { value: fake, writable: false, configurable: false, enumerable: false }); +for (const name of ["BroadcastChannel", "localStorage", "sessionStorage"]) delete globalThis[name]; + +// Downloaded code runs only through here, with options of no prototype (nothing read from Object.prototype). +const evaluate = (text, filename) => { + const options = Object.create(null); + options.filename = String(filename); + return runInThisContext(new Script(String(text), options)); +}; + +// Frames in: the first is the start (the bridge's start and the base bundle), the rest the launcher's messages. +let listener; +let started = false; +// Unread bytes: one buffer, grown by doubling, each byte copied in once. +let pending = BufferAlloc(64 * 1024); +let start = 0; +let end = 0; +const received = (payload) => { + const headerLength = readUInt32BE(payload, 0); + const header = parse(bufferText(payload, "utf8", 4, 4 + headerLength)); + const body = subarray(payload, 4 + headerLength); + if (!started) { + started = true; + run(header.start, bufferText(body, "utf8")); + return; + } + if (header.kind === "input") header.data = bufferText(body, "utf8"); + else if (header.kind === "socket_message" || header.kind === "attached") header.bytes = new Uint8Array(body); + if (listener !== undefined) listener(header); +}; +onInput(input, "data", (chunk) => { + if (end + chunk.length > pending.length) { + const unread = end - start; + const target = unread + chunk.length <= pending.length ? pending : BufferAlloc(Math.max(unread + chunk.length, pending.length * 2)); + copyBytes(pending, target, 0, start, end); + pending = target; + start = 0; + end = unread; + } + copyBytes(chunk, pending, end); + end += chunk.length; + while (end - start >= 4) { + const length = readUInt32BE(pending, start); + if (length > MAX_FRAME) apply(exit, real, [70]); + if (end - start < 4 + length) break; + // A copy: the buffer is reused, and what the frame carries is kept by the sandbox. + const payload = BufferAlloc(length); + copyBytes(pending, payload, 0, start + 4, start + 4 + length); + start += 4 + length; + received(payload); + } + if (start === end) start = end = 0; +}); +onInput(input, "end", () => apply(exit, real, [0])); +const run = (start, source) => { + globalThis.chordBridge = freeze({ + start: freeze(start), + send, + onMessage: (next) => { + listener = next; + }, + evaluate, + }); + try { + const base = { exports: {} }; + evaluate("(function (require, module, exports) {" + source + "\\n})", "chord-base.js")(builtin, base, base.exports); + } catch (error) { + send({ kind: "failed", message: String(error && error.message) }); + } +}; +`; diff --git a/packages/chord/src/next-react/README.md b/packages/chord/src/next-react/README.md new file mode 100644 index 000000000..11ee4c3af --- /dev/null +++ b/packages/chord/src/next-react/README.md @@ -0,0 +1,14 @@ +# react + +React bindings (slice 11 §3, decision "Slice 11, P7"): `@earendil-works/chord/next/react` once v2 has its entry point; +optional (React is an optional peer), non-React apps never load it. No components, no styling. + +- `HostProvider({ host })`, `useHost()`. +- `useHold(Type, params)`: a reference held while the component is mounted, released on unmount, held again when its + params change (compared by value); `undefined` on the first render. +- `useWatch(watchable)`: its current value (`undefined` while it has none), re-rendering on change + (`useSyncExternalStore` over `watchStore`). +- `useRegistry(R)`: the registry's composed value; `useRefState(ref)`: the reference's state. +- A component unmounted by a reload has not left (design §21): its releases are plain releases. + +Tests: `test/next-node/react/` (`stores.test.ts` in Node, `hooks.test.ts` in a real browser). diff --git a/packages/chord/src/next-react/hooks.ts b/packages/chord/src/next-react/hooks.ts new file mode 100644 index 000000000..a3731a920 --- /dev/null +++ b/packages/chord/src/next-react/hooks.ts @@ -0,0 +1,79 @@ +// React bindings (slice 11 §3, decision "Slice 11, P7"; p2's `useObject`, `useReadable`, `useRegistry`): a host in +// context, references held for a component's life, and watchables read with `useSyncExternalStore`. A component +// unmounted by a reload has not left (design §21): its releases are plain releases, presence is the app's. No +// components, no styling. + +import { + createContext, + createElement, + type ReactNode, + useContext, + useEffect, + useMemo, + useState, + useSyncExternalStore, +} from "react"; +import { BACKGROUND_CONTEXT, withAbortSignal } from "../context/index.ts"; +import type { Contract, ParamsOf } from "../next/contracts/index.ts"; +import type { Principal } from "../next/data/index.ts"; +import { canonicalJson } from "../next/data/principal.ts"; +import type { Held, Host, RefState } from "../next/host/index.ts"; +import type { Registry } from "../next/registries/index.ts"; +import type { NativeWatchable, Watchable } from "../watch.ts"; +import { watchStore } from "./stores.ts"; + +/** The host components use, given by `HostProvider`. */ +const HostContext = createContext | undefined>(undefined); + +/** Gives `host` to the components below. */ +export function HostProvider

(props: { readonly host: Host

; readonly children?: ReactNode }) { + // Components hold for the app's principal type: the context keeps it as the host was given. + return createElement(HostContext.Provider, { value: props.host as unknown as Host }, props.children); +} + +/** The host `HostProvider` gives; throws outside one. */ +export function useHost

(): Host

{ + const host = useContext(HostContext); + if (host === undefined) throw new Error("useHost: no HostProvider above this component"); + return host as unknown as Host

; +} + +/** + * A reference to `Type` with `params`, held while the component is mounted and released when it unmounts or its params + * change (compared by value); `undefined` before its first hold (the first render). + */ +export function useHold(Type: C, params: ParamsOf): Held | undefined { + const host = useHost(); + const key = canonicalJson(params as never, BACKGROUND_CONTEXT); + const [held, setHeld] = useState<{ readonly key: string; readonly ref: Held } | undefined>(undefined); + // biome-ignore lint/correctness/useExhaustiveDependencies: `key` is `params` by value: an inline object must not re-hold. + useEffect(() => { + const ref = host.hold(Type, params, BACKGROUND_CONTEXT); + setHeld({ key, ref }); + return () => ref.release(); + }, [host, Type, key]); + return held !== undefined && held.key === key ? held.ref : undefined; +} + +/** The current value of `watchable` (a state member, a derive), re-rendering on change; `undefined` while it has none. */ +export function useWatch(watchable: Watchable | NativeWatchable | undefined): T | undefined { + const store = useMemo(() => watchStore(watchable), [watchable]); + return useSyncExternalStore(store.subscribe, store.snapshot, store.snapshot); +} + +/** Registry `R`'s composed value on the component's host, re-rendering on change; `undefined` until it is usable. */ +export function useRegistry(R: Registry): V | undefined { + const host = useHost(); + const [value, setValue] = useState | undefined>(undefined); + useEffect(() => { + const abort = new AbortController(); + host.registry(R, withAbortSignal(abort.signal, BACKGROUND_CONTEXT)).then(setValue, () => {}); + return () => abort.abort(); + }, [host, R]); + return useWatch(value); +} + +/** The state of `ref` (loading, live, failed, ended), re-rendering on change; `undefined` without a reference. */ +export function useRefState(ref: { readonly state: NativeWatchable } | undefined): RefState | undefined { + return useWatch(ref?.state); +} diff --git a/packages/chord/src/next-react/index.ts b/packages/chord/src/next-react/index.ts new file mode 100644 index 000000000..03a78df79 --- /dev/null +++ b/packages/chord/src/next-react/index.ts @@ -0,0 +1,5 @@ +// Chord v2's React bindings (`@earendil-works/chord/next/react` once v2 has its entry point; slice 11 §3): optional, +// non-React apps never load them. + +export { HostProvider, useHold, useHost, useRefState, useRegistry, useWatch } from "./hooks.ts"; +export { type ExternalStore, watchStore } from "./stores.ts"; diff --git a/packages/chord/src/next-react/stores.ts b/packages/chord/src/next-react/stores.ts new file mode 100644 index 000000000..4b9e47e09 --- /dev/null +++ b/packages/chord/src/next-react/stores.ts @@ -0,0 +1,45 @@ +// External stores over Chord's watchables (slice 11 §3): what React's `useSyncExternalStore` reads. A store's snapshot +// is the watchable's current value (`undefined` while it has none: a reference not bound yet, ended, or a member its +// variant lacks; nothing throws during a render); subscribing starts a watch whose every call notifies React, and the +// unsubscribe ends it. + +import { BACKGROUND_CONTEXT, withAbortSignal } from "../context/index.ts"; +import type { NativeWatchable, Watchable } from "../watch.ts"; + +/** What `useSyncExternalStore` takes: subscribe (returning its unsubscribe) and the current snapshot. */ +export type ExternalStore = { + readonly subscribe: (changed: () => void) => () => void; + readonly snapshot: () => T | undefined; +}; + +/** The store of `source` (a state member, a derive, a registry value, `ref.state`); none for `undefined`. */ +export function watchStore(source: Watchable | NativeWatchable | undefined): ExternalStore { + return { + subscribe: (changed) => { + if (source === undefined) return () => {}; + const abort = new AbortController(); + const context = withAbortSignal(abort.signal, BACKGROUND_CONTEXT); + // A watch that cannot start (its source ended, the abort came first) only means no more changes. + source.watch(context).then( + (watch) => { + (watch.start as (listener: () => Promise) => void)(async () => { + changed(); + }); + // A watch starts at the value it acquired, which may be newer than the snapshot React read (a + // reference's members bind after the first render): its calls bring only later changes. + changed(); + }, + () => {}, + ); + return () => abort.abort(); + }, + snapshot: () => { + if (source === undefined) return undefined; + try { + return source.value; + } catch { + return undefined; + } + }, + }; +} diff --git a/packages/chord/src/next-tui/README.md b/packages/chord/src/next-tui/README.md new file mode 100644 index 000000000..31a5f5f8f --- /dev/null +++ b/packages/chord/src/next-tui/README.md @@ -0,0 +1,29 @@ +# tui (sandbox side) + +The terminal client's code inside the sandbox (slice 11 §4, decisions "Slice 11, P8" and "P9"): +`@earendil-works/chord/next/tui` once v2 has its entry point. The launcher is `next-node/tui`, the server side +`next-node/surfaces` (`tuiSurface`). + +```ts +// the app's terminal entry (its base bundle) +startTerminalRuntime

({ + modules: { "@acme/app/v3": api, "@acme/app/v3/tui": apiTui }, + core: [{ name: "@acme/tui", facet: tuiCore }], + render: ({ host, terminal, theme, copy, open, attach, exit }) => { + const scope = new TuiScope(host, () => redraw()); // hold, watch, registry; dispose + terminal.onInput((data) => …); terminal.write(frame); + theme.fg("accent", "Notes"); // colours by role + }, +}); +``` + +- `bridge.ts`: the messages between the sandbox and the launcher (input, resizes, relayed link sockets, requests with + replies: a plugin script, the server's base, `copy`, `open`; `outdated`, `exit`, `failed`). The bootstrap leaves the + bridge on `globalThis.chordBridge`; the runtime takes it off before any plugin runs. +- `channel.ts`: the sandbox's side: requests, `RelayLink` (a Chord link whose bytes the launcher relays), input and + resizes. `terminal.ts`: the bridged terminal (size, write, input, resize). `theme.ts`: the theme lookup (`fg`, `bg`, + `onChange`). `runtime.ts`: the client (`startClient` with `TuiScripts`; scripts run with `vm`; no wakes). +- `scope.ts`: `TuiScope(host, changed)`: `hold`, `watch` (a reader, `undefined` while no value), `registry`, `dispose` + (no `changed` after it). References wait and rebind by themselves; no terminal library dependency. + +Tests: `test/next/tui/` and, with the launcher and sandbox, `test/next-node/tui/`. diff --git a/packages/chord/src/next-tui/bridge.ts b/packages/chord/src/next-tui/bridge.ts new file mode 100644 index 000000000..8de935383 --- /dev/null +++ b/packages/chord/src/next-tui/bridge.ts @@ -0,0 +1,69 @@ +// The terminal's bridge (slice 11 §4, decision "Slice 11, P8"): the only way between the sandbox, where the app's +// terminal code and the plugins run, and the trusted launcher outside it. The launcher sends input, resizes and the +// relayed bytes of the Chord link; the sandbox sends frames for the terminal, opens and feeds link sockets through the +// launcher, asks for plugin scripts and the server's base hash, and asks for actions on the machine (copy, open a URL, +// attach a file the user names), which the launcher confirms with the user. Messages are plain data, framed over the +// sandbox's pipes (`next-node/tui/pipe.ts`); requests carry an id their reply has. + +/** What the launcher tells the sandbox at its start. */ +export type BridgeStart = { + /** The base bundle's hash (the sandbox runs it). */ + readonly base: string; + /** The text of the combined plugin scripts at start. */ + readonly combined: string; + readonly columns: number; + readonly rows: number; + /** How many colours the terminal shows. */ + readonly colors: "truecolor" | "256"; + /** The terminal's background: which of the user's two themes applies. */ + readonly mode: "dark" | "light"; +}; + +/** A request of the sandbox the launcher answers. */ +export type BridgeRequest = + | { readonly kind: "fetch"; readonly path: string } + | { readonly kind: "base" } + | { readonly kind: "copy"; readonly text: string } + | { readonly kind: "open"; readonly url: string } + /** A file of the machine the user names (`path`): its name and bytes once confirmed, else `null`. */ + | { readonly kind: "attach"; readonly path: string }; + +/** What the sandbox sends. */ +export type SandboxMessage = + | { readonly kind: "write"; readonly data: string } + | { readonly kind: "socket_open"; readonly socket: number } + | { readonly kind: "socket_send"; readonly socket: number; readonly bytes: Uint8Array } + | { readonly kind: "socket_close"; readonly socket: number } + | { readonly kind: "request"; readonly id: number; readonly request: BridgeRequest } + /** The sandbox's code is outdated (a new base bundle): the launcher starts a new one. */ + | { readonly kind: "outdated" } + /** The base bundle failed to start (its message; the launcher shows none of the sandbox's own output). */ + | { readonly kind: "failed"; readonly message: string } + | { readonly kind: "exit"; readonly code: number }; + +/** What the launcher sends. */ +export type LauncherMessage = + | { readonly kind: "input"; readonly data: string } + | { readonly kind: "resize"; readonly columns: number; readonly rows: number } + | { readonly kind: "socket_opened"; readonly socket: number } + | { readonly kind: "socket_message"; readonly socket: number; readonly bytes: Uint8Array } + | { readonly kind: "socket_closed"; readonly socket: number } + | { readonly kind: "reply"; readonly id: number; readonly value: unknown } + /** An attach request's answer: the file the user confirmed. */ + | { readonly kind: "attached"; readonly id: number; readonly name: string; readonly bytes: Uint8Array } + | { readonly kind: "refused"; readonly id: number; readonly message: string }; + +/** The bridge as the sandbox's code gets it from the bootstrap (once: it is taken off `globalThis`). */ +export type Bridge = { + readonly start: BridgeStart; + send(message: SandboxMessage): void; + onMessage(listener: (message: LauncherMessage) => void): void; + /** + * Runs `source` (registered plugin scripts) in the sandbox's context, as the bootstrap ran the base bundle: with no + * dynamic import callback (`import()` throws). + */ + evaluate(source: string, filename: string): void; +}; + +/** The global the sandbox's bootstrap leaves the bridge on, for the base bundle's runtime to take. */ +export const BRIDGE = "chordBridge"; diff --git a/packages/chord/src/next-tui/channel.ts b/packages/chord/src/next-tui/channel.ts new file mode 100644 index 000000000..d7ed7755c --- /dev/null +++ b/packages/chord/src/next-tui/channel.ts @@ -0,0 +1,163 @@ +// The sandbox's side of the bridge (slice 11 §4): one dispatcher of the launcher's messages (input, resizes, link +// sockets, replies), requests with their replies, and Chord links whose bytes the launcher relays to the server. + +import { LinkEnd } from "../next/link/link.ts"; +import type { Bridge, BridgeRequest, BridgeStart, LauncherMessage } from "./bridge.ts"; +import { splitFrames } from "./frames.ts"; + +/** What one relayed socket reports: it opened, a frame arrived. */ +export type RelayEvents = { opened(): void; received(): void }; + +/** The sandbox's bridge: requests, relayed sockets, and the terminal's input and resizes. */ +export class SandboxChannel { + readonly #bridge: Bridge; + readonly #replies = new Map(); + readonly #sockets = new Map(); + readonly #input = new Set<(data: string) => void>(); + readonly #resize = new Set<(columns: number, rows: number) => void>(); + #next = 0; + + constructor(bridge: Bridge) { + this.#bridge = bridge; + bridge.onMessage((message) => this.#dispatch(message)); + } + + /** What the launcher told at the start. */ + get start(): BridgeStart { + return this.#bridge.start; + } + + /** Asks the launcher; rejects with its refusal. */ + request(request: BridgeRequest): Promise { + const id = ++this.#next; + return new Promise((resolve, reject) => { + this.#replies.set(id, { resolve, reject }); + this.#bridge.send({ kind: "request", id, request }); + }); + } + + /** Runs the registered scripts `text` (named `filename` in stacks) through the sandbox's own evaluation. */ + evaluate(text: string, filename: string): void { + this.#bridge.evaluate(text, filename); + } + + /** Writes `data` to the terminal, in frames the launcher takes (`splitFrames`). */ + write(data: string): void { + for (const frame of splitFrames(data)) this.#bridge.send({ kind: "write", data: frame }); + } + + /** Tells the launcher this sandbox's code is outdated. */ + outdated(): void { + this.#bridge.send({ kind: "outdated" }); + } + + /** Ends the terminal with `code`. */ + exit(code: number): void { + this.#bridge.send({ kind: "exit", code }); + } + + /** Calls `listener` with each input (keys, pastes) until the returned removal. */ + onInput(listener: (data: string) => void): () => void { + this.#input.add(listener); + return () => this.#input.delete(listener); + } + + /** Calls `listener` at each resize until the returned removal. */ + onResize(listener: (columns: number, rows: number) => void): () => void { + this.#resize.add(listener); + return () => this.#resize.delete(listener); + } + + /** Opens a socket to the server's links endpoint through the launcher: its link, reporting to `events`. */ + openSocket(events: RelayEvents): RelayLink { + const socket = ++this.#next; + const link = new RelayLink(socket, events, this.#bridge, () => this.#sockets.delete(socket)); + this.#sockets.set(socket, link); + this.#bridge.send({ kind: "socket_open", socket }); + return link; + } + + /** One message of the launcher, to whom it is for. */ + #dispatch(message: LauncherMessage): void { + switch (message.kind) { + case "input": + for (const listener of this.#input) listener(message.data); + return; + case "resize": + for (const listener of this.#resize) listener(message.columns, message.rows); + return; + case "socket_opened": + this.#sockets.get(message.socket)?.opened(); + return; + case "socket_message": + this.#sockets.get(message.socket)?.received(message.bytes); + return; + case "socket_closed": + this.#sockets.get(message.socket)?.dropped(); + return; + case "reply": + case "attached": + case "refused": + this.#settle(message); + } + } + + /** Settles the request a reply or refusal answers. */ + #settle(message: Extract): void { + const waiting = this.#replies.get(message.id); + this.#replies.delete(message.id); + if (message.kind === "reply") waiting?.resolve(message.value); + else if (message.kind === "attached") waiting?.resolve({ name: message.name, bytes: message.bytes }); + else waiting?.reject(new Error(message.message)); + } +} + +/** A Chord link over a socket the launcher holds: bytes relayed both ways, buffered until the socket opened. */ +export class RelayLink extends LinkEnd { + readonly #socket: number; + readonly #events: RelayEvents; + readonly #bridge: Bridge; + readonly #forget: () => void; + /** Messages sent while the socket connects; `undefined` once open. */ + #connecting: Uint8Array[] | undefined = []; + + constructor(socket: number, events: RelayEvents, bridge: Bridge, forget: () => void) { + super(); + this.#socket = socket; + this.#events = events; + this.#bridge = bridge; + this.#forget = forget; + } + + send(message: Uint8Array): void { + if (this.closed) return; + if (this.#connecting !== undefined) this.#connecting.push(message); + else this.#bridge.send({ kind: "socket_send", socket: this.#socket, bytes: message }); + } + + close(): void { + if (this.closed) return; + this.#bridge.send({ kind: "socket_close", socket: this.#socket }); + this.dropped(); + } + + /** The launcher's socket opened: what was sent meanwhile goes out. */ + opened(): void { + const queued = this.#connecting ?? []; + this.#connecting = undefined; + this.#events.opened(); + for (const message of queued) this.send(message); + } + + /** A frame arrived. */ + received(bytes: Uint8Array): void { + this.#events.received(); + this.deliver(bytes); + } + + /** The socket closed (either side): the link ends. */ + dropped(): void { + this.#forget(); + this.ended(); + } +} diff --git a/packages/chord/src/next-tui/frames.ts b/packages/chord/src/next-tui/frames.ts new file mode 100644 index 000000000..29f3d5d91 --- /dev/null +++ b/packages/chord/src/next-tui/frames.ts @@ -0,0 +1,51 @@ +// Frames for the terminal (slice 11 §4; fix round 1, Fable L9): the launcher drops a frame over 64 KiB whole (an escape +// it could hide is never written in parts), so the sandbox splits a large write into frames, each cut only between +// escape sequences and never inside a surrogate pair: a first paint of a large true-colour screen arrives whole. + +/** The most UTF-16 units of one frame: at most 3 UTF-8 bytes each, so under the launcher's 64 KiB. */ +export const FRAME_UNITS = 16 * 1024; + +/** `data` as frames of at most `FRAME_UNITS` units, cut outside escape sequences (one longer is its own frame). */ +export function splitFrames(data: string): string[] { + if (data.length <= FRAME_UNITS) return [data]; + const frames: string[] = []; + let start = 0; + // The last place a frame may end: after a character or a whole sequence, never inside one. + let safe = 0; + let index = 0; + while (index < data.length) { + const end = data.charCodeAt(index) === 27 ? sequenceEnd(data, index) : unitEnd(data, index); + if (end - start > FRAME_UNITS && safe > start) { + frames.push(data.slice(start, safe)); + start = safe; + } + index = end; + safe = end; + } + frames.push(data.slice(start)); + return frames; +} + +/** The index after the character at `index`, a surrogate pair kept together. */ +function unitEnd(data: string, index: number): number { + const code = data.charCodeAt(index); + return code >= 0xd800 && code <= 0xdbff && index + 1 < data.length ? index + 2 : index + 1; +} + +/** The index after the escape sequence at `index` (CSI to its final; OSC, DCS and the like to their terminator). */ +function sequenceEnd(data: string, index: number): number { + const introducer = data[index + 1]; + if (introducer === "[") { + let at = index + 2; + while (at < data.length && data.charCodeAt(at) >= 0x20 && data.charCodeAt(at) <= 0x3f) at++; + return Math.min(at + 1, data.length); + } + if (introducer !== undefined && "]PX^_".includes(introducer)) { + for (let at = index + 2; at < data.length; at++) { + if (data.charCodeAt(at) === 7) return at + 1; + if (data[at] === "\x1b" && data[at + 1] === "\\") return at + 2; + } + return data.length; + } + return Math.min(index + 2, data.length); +} diff --git a/packages/chord/src/next-tui/index.ts b/packages/chord/src/next-tui/index.ts new file mode 100644 index 000000000..fd5db3903 --- /dev/null +++ b/packages/chord/src/next-tui/index.ts @@ -0,0 +1,14 @@ +// Chord v2's terminal surface, the sandbox side (`@earendil-works/chord/next/tui` once v2 has its entry point; slice +// 11 §4): `startTerminalRuntime` in the app's terminal entry, the bridged terminal, the theme lookup and `TuiScope`. +// The launcher and the server side are `next/tui/launcher` and `next/web/server`'s `tuiSurface`. + +export type { Bridge, BridgeRequest, BridgeStart, LauncherMessage, SandboxMessage } from "./bridge.ts"; +export { + type AttachedFile, + startTerminalRuntime, + type TerminalRuntime, + type TerminalRuntimeOptions, +} from "./runtime.ts"; +export { TuiScope } from "./scope.ts"; +export { BridgeTerminal } from "./terminal.ts"; +export { TerminalTheme } from "./theme.ts"; diff --git a/packages/chord/src/next-tui/runtime.ts b/packages/chord/src/next-tui/runtime.ts new file mode 100644 index 000000000..d89e34f7c --- /dev/null +++ b/packages/chord/src/next-tui/runtime.ts @@ -0,0 +1,155 @@ +// `startTerminalRuntime` (slice 11 §4, decisions "Slice 11, P8" and "P9"): the terminal client inside the sandbox, in +// the app's terminal entry (the base bundle the launcher downloaded). A client host like a page's: the plugin scripts +// the launcher brought, loaded with the app's core facets and the kit's themes; the link to the server relayed by the +// launcher, kept by the same connection loop; `TuiScripts` followed to apply new plugin scripts in place; a new base +// bundle hands over to the launcher, which starts the new one. The app gets the host, the bridged terminal, the theme +// lookup and the launcher's actions. + +import { BACKGROUND_CONTEXT } from "../context/index.ts"; +import type { Principal } from "../next/data/index.ts"; +import type { PluginFacet } from "../next/host/index.ts"; +import { + type Client, + type ClientError, + type ClientModules, + type ConnectionState, + FALLBACK_THEMES, + type FacetFunction, + followActiveTheme, + REGISTER, + type RegisteredScript, + startClient, + TuiScripts, + themesFacet, +} from "../next/surfaces/index.ts"; +import { BRIDGE, type Bridge } from "./bridge.ts"; +import { SandboxChannel } from "./channel.ts"; +import { BridgeTerminal } from "./terminal.ts"; +import { TerminalTheme } from "./theme.ts"; + +/** What `startTerminalRuntime` takes. */ +export type TerminalRuntimeOptions

= { + /** The modules the terminal plugins import, by specifier: the app's API modules for the kind, its components. */ + readonly modules: ClientModules; + /** The app's own terminal facets. */ + readonly core?: readonly PluginFacet

[]; + readonly canRetry?: () => Promise; + readonly onConnection?: (state: ConnectionState) => void; + /** Draws the app, once its plugins are loaded. */ + readonly render: (runtime: TerminalRuntime

) => void; + /** Plugins left out or kept old, a failed update, the host's errors; dropped by default (stderr is the launcher's). */ + readonly onError?: (error: ClientError) => void; +}; + +/** A file the launcher read for the sandbox: its name (the path's last segment) and its bytes. */ +export type AttachedFile = { readonly name: string; readonly bytes: Uint8Array }; + +/** What the app's terminal code gets. */ +export type TerminalRuntime

= { + readonly host: Client

["host"]; + readonly terminal: BridgeTerminal; + readonly theme: TerminalTheme; + /** Asks the launcher to copy `text` to the machine's clipboard; it asks the user. Whether it was done. */ + copy(text: string): Promise; + /** Asks the launcher to open `url` (http or https) in a browser; it asks the user. Whether it was done. */ + open(url: string): Promise; + /** + * Asks the launcher for the file at `path` on the machine (at most 16 MiB); it asks the user. Its name and bytes, or + * `undefined` when declined. + */ + attach(path: string): Promise; + /** Ends the terminal client with `code`. */ + exit(code: number): void; +}; + +/** Starts the terminal client inside the sandbox; resolves with its runtime once rendered. */ +export async function startTerminalRuntime

( + options: TerminalRuntimeOptions

, +): Promise> { + const channel = new SandboxChannel(takeBridge()); + const { start } = channel; + const scripts = new SandboxScripts(channel); + const core: PluginFacet

[] = [{ name: "chord.themes", facet: themesFacet

() }, ...(options.core ?? [])]; + const client = await startClient

({ + scripts: TuiScripts, + base: start.base, + initial: scripts.run(start.combined), + modules: options.modules, + core, + open: (events) => channel.openSocket(events), + fetchScript: async (entry) => { + const text = (await channel.request({ kind: "fetch", path: entry.url })) as string; + const registered = scripts + .run(text) + .find(({ plugin, digest }) => plugin === entry.name && digest === entry.digest); + if (registered === undefined) throw new Error(`plugin "${entry.name}": ${entry.url} did not register`); + return registered; + }, + serverBase: () => + channel.request({ kind: "base" }).then( + (base) => base as string, + () => undefined, + ), + outdated: () => channel.outdated(), + ...(options.canRetry === undefined ? {} : { canRetry: options.canRetry }), + ...(options.onConnection === undefined ? {} : { onConnection: options.onConnection }), + environment: TERMINAL_ENVIRONMENT, + onError: options.onError ?? (() => {}), + }); + const theme = new TerminalTheme(FALLBACK_THEMES[start.mode], start.colors); + await followActiveTheme(client.host, start.mode, (active) => theme.set(active), BACKGROUND_CONTEXT); + const runtime: TerminalRuntime

= { + host: client.host, + terminal: new BridgeTerminal(channel), + theme, + copy: (text) => channel.request({ kind: "copy", text }).then(Boolean), + open: (url) => channel.request({ kind: "open", url }).then(Boolean), + // The launcher's reply: its own data, `null` when declined. + attach: (path) => + channel.request({ kind: "attach", path }).then((file) => (file as AttachedFile | null) ?? undefined), + exit: (code) => channel.exit(code), + }; + options.render(runtime); + return runtime; +} + +/** The bridge the bootstrap left on `globalThis`, taken off it (plugin code never finds it there). */ +function takeBridge(): Bridge { + const scope = globalThis as Record; + const bridge = scope[BRIDGE] as Bridge | undefined; + if (bridge === undefined) throw new Error("startTerminalRuntime: not inside the terminal's sandbox (no bridge)"); + delete scope[BRIDGE]; + return bridge; +} + +/** A terminal has no page wakes: the loop retries on its backoff alone. */ +const TERMINAL_ENVIRONMENT = { + now: () => Date.now(), + after: (ms: number, callback: () => void) => { + const timer = setTimeout(callback, ms); + return () => clearTimeout(timer); + }, + online: () => true, + onWake: () => () => {}, +}; + +/** Runs registered scripts' text in the sandbox and collects what registered (`chordFacet`). */ +class SandboxScripts { + readonly #channel: SandboxChannel; + /** The scripts the text being run registers. */ + #collected: RegisteredScript[] = []; + + constructor(channel: SandboxChannel) { + this.#channel = channel; + (globalThis as Record)[REGISTER] = (plugin: string, digest: string, run: FacetFunction) => { + this.#collected.push({ plugin, digest, run }); + }; + } + + /** Runs `text` (one or more registered scripts): what it registered. */ + run(text: string): RegisteredScript[] { + this.#collected = []; + this.#channel.evaluate(text, "chord-plugins.js"); + return this.#collected; + } +} diff --git a/packages/chord/src/next-tui/scope.ts b/packages/chord/src/next-tui/scope.ts new file mode 100644 index 000000000..8302e4d02 --- /dev/null +++ b/packages/chord/src/next-tui/scope.ts @@ -0,0 +1,81 @@ +// `TuiScope` (slice 11 §4, decision "Slice 11, P9"; p2's `TuiScope`): a terminal component's Chord resources, the +// counterpart of the React bindings for terminal UI libraries. It holds references, follows watchables and registries, +// calls one `changed` callback (the app's redraw) on every change, and releases everything at `dispose`. References +// wait and rebind by themselves (no retry loop, unlike p2's); no terminal library dependency. + +import { BACKGROUND_CONTEXT, withAbortSignal } from "../context/index.ts"; +import type { Contract, ParamsOf } from "../next/contracts/index.ts"; +import type { Principal } from "../next/data/index.ts"; +import type { Held, Host } from "../next/host/index.ts"; +import type { Registry } from "../next/registries/index.ts"; +import type { NativeWatchable, Watchable } from "../watch.ts"; + +/** A component's scope: what it holds and follows, until `dispose`. */ +export class TuiScope

{ + readonly #host: Host

; + readonly #changed: () => void; + readonly #abort = new AbortController(); + readonly #releases: (() => void)[] = []; + + constructor(host: Host

, changed: () => void) { + this.#host = host; + this.#changed = changed; + } + + /** A reference to `Type` with `params`, held until `dispose`; its state changes call `changed`. */ + hold(Type: C, params: ParamsOf): Held { + const held = this.#host.hold(Type, params, this.#context()); + this.#releases.push(() => held.release()); + this.watch(held.state); + return held; + } + + /** + * Follows `source` (a member, a derive, `ref.state`) until `dispose`: its changes call `changed`. Returns a reader of + * its current value (`undefined` while it has none: nothing throws in a redraw). + */ + watch(source: Watchable | NativeWatchable): () => T | undefined { + source.watch(this.#context()).then( + (watch) => { + (watch.start as (listener: () => Promise) => void)(async () => { + if (!this.#abort.signal.aborted) this.#changed(); + }); + // The value it acquired may be newer than what was drawn (a reference bound since): draw it. + if (!this.#abort.signal.aborted) this.#changed(); + }, + // Disposed (or the source ended) before the watch started: nothing to follow. + () => {}, + ); + return () => { + try { + return source.value; + } catch { + return undefined; + } + }; + } + + /** Registry `R`'s composed value, followed until `dispose`; a reader, `undefined` until it is usable. */ + registry(R: Registry): () => V | undefined { + let read: () => V | undefined = () => undefined; + this.#host.registry(R, this.#context()).then( + (value) => { + read = this.watch(value); + if (!this.#abort.signal.aborted) this.#changed(); + }, + () => {}, + ); + return () => read(); + } + + /** Releases every reference and ends every watch; `changed` is not called after it. */ + dispose(): void { + this.#abort.abort(); + for (const release of this.#releases.splice(0)) release(); + } + + /** The context of the scope's work: aborted at `dispose`. */ + #context() { + return withAbortSignal(this.#abort.signal, BACKGROUND_CONTEXT); + } +} diff --git a/packages/chord/src/next-tui/terminal.ts b/packages/chord/src/next-tui/terminal.ts new file mode 100644 index 000000000..a29723cf4 --- /dev/null +++ b/packages/chord/src/next-tui/terminal.ts @@ -0,0 +1,45 @@ +// The terminal as the app's terminal code sees it inside the sandbox (slice 11 §4): its size, writing frames, and +// input and resizes, all through the bridge; the launcher holds the real terminal. Library-agnostic: an app adapts it +// to its terminal UI library (pi-tui's `Terminal` in a few lines). + +import type { SandboxChannel } from "./channel.ts"; + +/** The bridged terminal. */ +export class BridgeTerminal { + readonly #channel: SandboxChannel; + #columns: number; + #rows: number; + + constructor(channel: SandboxChannel) { + this.#channel = channel; + this.#columns = channel.start.columns; + this.#rows = channel.start.rows; + channel.onResize((columns, rows) => { + this.#columns = columns; + this.#rows = rows; + }); + } + + get columns(): number { + return this.#columns; + } + + get rows(): number { + return this.#rows; + } + + /** Writes `data` (text and escape sequences) to the terminal. */ + write(data: string): void { + this.#channel.write(data); + } + + /** Calls `listener` with each input (keys, pastes, as the terminal sends them) until the returned removal. */ + onInput(listener: (data: string) => void): () => void { + return this.#channel.onInput(listener); + } + + /** Calls `listener` after each resize (the new size read from `columns` and `rows`) until the returned removal. */ + onResize(listener: () => void): () => void { + return this.#channel.onResize(() => listener()); + } +} diff --git a/packages/chord/src/next-tui/theme.ts b/packages/chord/src/next-tui/theme.ts new file mode 100644 index 000000000..724356e3d --- /dev/null +++ b/packages/chord/src/next-tui/theme.ts @@ -0,0 +1,44 @@ +// The terminal's theme lookup (slice 11 §4b, decision "Looks and themes"): colours by role from the active theme, as +// the terminal shows them (true colour, else the nearest of 256; a theme's `tui` overrides first). A plugin asks for a +// role (`theme.fg("accent", "Notes")`), never writes raw colours; a new theme applies at the next redraw. + +import { type ColorDepth, type Theme, terminalColor } from "../next/surfaces/index.ts"; + +/** Colours by role for the terminal. */ +export class TerminalTheme { + readonly #depth: ColorDepth; + #theme: Theme; + readonly #listeners = new Set<() => void>(); + + constructor(theme: Theme, depth: ColorDepth) { + this.#theme = theme; + this.#depth = depth; + } + + /** The active theme. */ + get theme(): Theme { + return this.#theme; + } + + /** `text` in `role`'s colour (the default colour after it). */ + fg(role: string, text: string): string { + return `\x1b[${terminalColor(this.#theme, role, this.#depth, "fg")}m${text}\x1b[39m`; + } + + /** `text` on `role`'s colour as its background. */ + bg(role: string, text: string): string { + return `\x1b[${terminalColor(this.#theme, role, this.#depth, "bg")}m${text}\x1b[49m`; + } + + /** Calls `listener` when the active theme changes (to redraw) until the returned removal. */ + onChange(listener: () => void): () => void { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + /** The active theme is now `theme`. */ + set(theme: Theme): void { + this.#theme = theme; + for (const listener of this.#listeners) listener(); + } +} diff --git a/packages/chord/src/next-web/README.md b/packages/chord/src/next-web/README.md new file mode 100644 index 000000000..0c946b782 --- /dev/null +++ b/packages/chord/src/next-web/README.md @@ -0,0 +1,25 @@ +# web (browser side) + +`startPage` (slice 11 §2) for the app's web entry, the base bundle the server builds: `@earendil-works/chord/next/web` +once v2 has its entry point. The server side is `next-node/surfaces` (`webSurface`). + +```ts +startPage

({ + connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), + modules: { "@acme/app/v3": api, "@acme/app/v3/web": apiWeb, react: React, "react/jsx-runtime": jsx }, + core: [{ name: "@acme/web", facet: webCore }], + canRetry: async () => true, // false: stop reconnecting (a guest's ended share) + onConnection: (state) => setBanner(state), // connecting | connected | waiting + render: (host) => createRoot(root).render(), +}); +``` + +- The page's inline bootstrap (`PageAssets.bootstrap`, the response's nonce) defines `chordFacet` before the combined + file runs; its scripts queue, and `startPage` takes them (`scripts.ts`). Later scripts load with ``; + +export default defineFacet((host, self) => { + host.serve(Channels, () => ({ variant: "main", members: { names: constant(CHANNELS) } })); + self.contribute(Http, { + routes: { + "GET /": { public: true, handler: async (request) => ({ kind: "html", body: signIn(request.nonce) }) }, + "GET /files/:channel/:id": async (request, ctx) => { + const room = await ctx.get(Room, { id: request.params.channel! }); + const file = room.messages.value.find((message) => message.file?.id === request.params.id)?.file; + if (file === undefined) return { kind: "error", status: 404 }; + return { + kind: "bytes", + mime: file.mime, + size: file.size, + inline: file.mime.startsWith("image/") ? true : undefined, + download: file.mime.startsWith("image/") ? undefined : file.name, + read: (offset, length) => + room.read({ id: file.id, offset, length: Math.min(length, 1024 * 1024) }, BACKGROUND_CONTEXT), + }; + }, + }, + }); +}); diff --git a/packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts b/packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts new file mode 100644 index 000000000..7f78cf8cb --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/plugins/chat/worker.ts @@ -0,0 +1,78 @@ +// The chat plugin on a channel's worker: the room's messages in a document of the worker's store (kept by a local +// object: a persisted state needs a stable identity), and each principal's `Room` posting as them and sharing files +// through the kit's `Files` of this worker. + +import { + Attachment, + arr, + defineFacet, + defineState, + local, + Message, + method, + num, + obj, + opt, + Room, + str, +} from "@chat/app/v1"; +import { Files } from "@chat/app/v1/worker"; + +/** The newest messages a channel keeps. */ +const KEPT = 500; + +const Log = defineState("chat.log", { + schema: obj({ next: num(), messages: arr(Message) }), + version: 1, + initial: () => ({ next: 1, messages: [] }), + persist: true, + checkpointWhen: (_value, _ops, { deltasSinceBase }) => deltasSinceBase >= 50, +}); + +const Store = local("chat.store", { + version: 1, + params: {}, + members: { messages: arr(Message), add: method(obj({ author: str(), text: str(), file: opt(Attachment) })) }, +}); + +export default defineFacet((host) => { + host.serve(Store, (_params, self) => { + const log = self.state(Log); + return { + messages: self.derive([log], ([value]) => value.messages), + add: async ({ author, text, file }, ctx) => { + await log.change((draft) => { + draft.messages.push({ + id: draft.next++, + author, + text, + at: Date.now(), + ...(file === undefined ? {} : { file }), + }); + if (draft.messages.length > KEPT) draft.messages.splice(0, draft.messages.length - KEPT); + }, ctx); + return null; + }, + }; + }); + host.serve(Room, (_params, principal, self) => { + const store = self.get(Store, {}); + const files = self.get(Files, {}); + const author = (principal as { readonly name?: string }).name ?? principal.kind; + return { + variant: "main", + members: { + messages: self.derive([store.messages], ([messages]) => messages), + post: (text, ctx) => store.add({ author, text }, ctx), + begin: (file, ctx) => files.begin(file, ctx), + append: (chunk, ctx) => files.append(chunk, ctx), + share: async ({ upload, text }, ctx) => { + const id = await files.finish(upload, ctx); + const { name, mime, size } = files.list.value.find((file) => file.id === id)!; + return store.add({ author, text, file: { id, name, mime, size } }, ctx); + }, + read: (range, ctx) => files.read(range, ctx), + }, + }; + }); +}); diff --git a/packages/chord/test/next/examples/59-chat/plugins/login.server.ts b/packages/chord/test/next/examples/59-chat/plugins/login.server.ts new file mode 100644 index 000000000..c6a5a9779 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/plugins/login.server.ts @@ -0,0 +1,5 @@ +// The kit's pick-a-name login, added to the chat as a one-line plugin. + +import { login } from "@chat/app/v1/server"; + +export default login; diff --git a/packages/chord/test/next/examples/59-chat/start.ts b/packages/chord/test/next/examples/59-chat/start.ts new file mode 100644 index 000000000..304449280 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/start.ts @@ -0,0 +1,76 @@ +// The chat's start: one server keeping the channel list and the surfaces, one worker per channel, a web page at `/chat` +// (the sign-in page at `/` comes from the chat plugin) and a terminal client. No harness: the app is its API, two +// plugins and its two client entries. + +import { cp, mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Context } from "../../../../src/context/index.ts"; +import { type App, startServer } from "../../../../src/next-node/app/index.ts"; +import { type PageAssets, pageScripts } from "../../../../src/next-node/surfaces/index.ts"; +import type { ChatPrincipal } from "./app/principals.ts"; +import * as v1server from "./app/v1/server.ts"; +import * as v1worker from "./app/v1/worker.ts"; +import * as v1 from "./app/v1.ts"; + +/** The chat's API, as the server and the channel workers have it. */ +export const API = { + name: "@chat/app", + versions: ["v1"], + modules: { "@chat/app/v1": v1, "@chat/app/v1/server": v1server, "@chat/app/v1/worker": v1worker }, +}; + +/** The chat's facet kinds. */ +export const KINDS = { + server: { platform: "node" }, + worker: { platform: "node" }, + web: { platform: "browser", packages: { react: {}, "react/jsx-runtime": {} } }, + tui: { platform: "node" }, +} as const; + +/** The chat page: the app's template around the surface's assets. */ +function page(request: { readonly nonce: string }, assets: PageAssets): string { + return `Chat${pageScripts(assets, request.nonce)}

`; +} + +/** A data directory and a copy of the chat's plugins, in a new temporary directory. */ +export async function chatDirectories(): Promise<{ + readonly base: string; + readonly dataDir: string; + readonly root: string; +}> { + const base = await mkdtemp(join(tmpdir(), "chat-")); + const root = join(base, "plugins"); + await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + return { base, dataDir: join(base, "data"), root }; +} + +/** Starts the chat on a free loopback port. */ +export function startChat(dataDir: string, root: string, context: Context): Promise> { + return startServer( + { + name: "chat", + dataDir, + root, + listen: { host: "127.0.0.1", port: 0 }, + api: API, + kinds: KINDS, + principal: { server: { kind: "server" }, worker: (address) => ({ kind: "worker", channel: address.key }) }, + workers: [ + { + kind: v1.Channel, + entry: fileURLToPath(new URL("./worker.ts", import.meta.url)), + execArgv: ["--conditions=source", "--experimental-strip-types", "--no-warnings"], + start: () => ({}), + // Anyone signed in may join a channel. + authorize: (principal) => principal.kind === "user" || principal.kind === "owner", + }, + ], + web: { entry: fileURLToPath(new URL("./web/main.ts", import.meta.url)), page, paths: ["/chat"] }, + tui: { entry: fileURLToPath(new URL("./tui/main.ts", import.meta.url)) }, + onError: (error) => console.error("chat:", error.kind, error.error), + }, + context, + ); +} diff --git a/packages/chord/test/next/examples/59-chat/tui/main.ts b/packages/chord/test/next/examples/59-chat/tui/main.ts new file mode 100644 index 000000000..b3dd20f52 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/tui/main.ts @@ -0,0 +1,90 @@ +// The chat in a terminal (the terminal base bundle, run in the launcher's sandbox): the channels (Tab switches), the +// current room's messages (an image as a placeholder, other files as their link), and a line to type into. Enter +// posts; `/attach ` asks the launcher for a file of this machine (it asks the user) and shares it in chunks; +// Ctrl-C quits. + +import type { Held } from "../../../../../src/next/index.ts"; +import { type AttachedFile, startTerminalRuntime, TuiScope } from "../../../../../src/next-tui/index.ts"; +import * as v1 from "../app/v1.ts"; + +const { BACKGROUND_CONTEXT: context, Channels, Room } = v1; + +/** The size of one upload chunk. */ +const CHUNK = 256 * 1024; + +/** Shares `file` in `room`, in chunks. */ +async function share(room: Held, file: AttachedFile): Promise { + const mime = /\.(png|jpe?g|gif|webp)$/i.test(file.name) + ? `image/${file.name.split(".").pop()!.toLowerCase()}` + : "application/octet-stream"; + const id = await room.begin({ name: file.name, mime }, context); + for (let offset = 0; offset < file.bytes.length; offset += CHUNK) { + await room.append({ upload: id, offset, bytes: file.bytes.subarray(offset, offset + CHUNK) }, context); + } + await room.share({ upload: id, text: "" }, context); +} + +/** One message as a line. */ +function line(message: v1.Message, channel: string, theme: { fg(role: string, text: string): string }): string { + const { file } = message; + const attached = + file === undefined + ? "" + : file.mime.startsWith("image/") + ? ` ${theme.fg("muted", `[image: ${file.name}]`)}` + : ` ${theme.fg("accent", `[${file.name}] /files/${channel}/${file.id}`)}`; + return `${theme.fg("accent", message.author)} ${message.text}${attached}`; +} + +void startTerminalRuntime({ + modules: { "@chat/app/v1": v1 }, + render: (runtime) => { + const { terminal, theme } = runtime; + const scope = new TuiScope(runtime.host, () => draw()); + const channels = scope.watch(scope.hold(Channels, {}).names); + // Each channel's room held once, on first view. + const rooms = new Map; messages: () => readonly v1.Message[] | undefined }>(); + const roomOf = (id: string) => { + let held = rooms.get(id); + if (held === undefined) { + const room = scope.hold(Room, { id }); + held = { room, messages: scope.watch(room.messages) }; + rooms.set(id, held); + } + return held; + }; + let channel = "general"; + let draft = ""; + const draw = (): void => { + const tabs = (channels() ?? []).map((name) => + name === channel ? theme.fg("accent", `#${name}`) : `#${name}`, + ); + const shown = (roomOf(channel).messages() ?? []) + .slice(-(terminal.rows - 3)) + .map((message) => line(message, channel, theme)); + terminal.write(`\x1b[2J\x1b[H${tabs.join(" ")}\r\n${shown.join("\r\n")}\r\n> ${draft}`); + }; + const send = (text: string): void => { + if (text.startsWith("/attach ")) { + const { room } = roomOf(channel); + void runtime + .attach(text.slice(8).trim()) + .then((file) => (file === undefined ? undefined : share(room, file))); + } else if (text !== "") void roomOf(channel).room.post(text, context); + }; + terminal.onInput((data) => { + if (data === "\x03") runtime.exit(0); + else if (data === "\t") { + const names = channels() ?? []; + channel = names[(names.indexOf(channel) + 1) % names.length] ?? channel; + } else if (data === "\r") { + send(draft); + draft = ""; + } else if (data === "\x7f") draft = draft.slice(0, -1); + else if (!data.startsWith("\x1b")) draft += data; + draw(); + }); + theme.onChange(draw); + draw(); + }, +}); diff --git a/packages/chord/test/next/examples/59-chat/web/main.ts b/packages/chord/test/next/examples/59-chat/web/main.ts new file mode 100644 index 000000000..478f2b218 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/web/main.ts @@ -0,0 +1,95 @@ +// The chat's page (the web base bundle): the channels as tabs, the current room's messages (images inline, other files +// as links), a message box and a file picker. A file goes up in chunks through the room (into its worker's files). + +import * as React from "react"; +import { createElement as h, useState } from "react"; +import * as jsx from "react/jsx-runtime"; +import { createRoot } from "react-dom/client"; +import type { Held } from "../../../../../src/next/index.ts"; +import { HostProvider, useHold, useWatch } from "../../../../../src/next-react/index.ts"; +import { startPage } from "../../../../../src/next-web/index.ts"; +import * as v1 from "../app/v1.ts"; +import "./style.css"; + +const { BACKGROUND_CONTEXT: context, Channels, Room } = v1; + +/** The size of one upload chunk. */ +const CHUNK = 256 * 1024; + +/** Sends `file` to `room` in chunks, then shares it. */ +async function upload(room: Held, file: File): Promise { + const id = await room.begin({ name: file.name, mime: file.type || "application/octet-stream" }, context); + for (let offset = 0; offset < file.size; offset += CHUNK) { + const bytes = new Uint8Array(await file.slice(offset, offset + CHUNK).arrayBuffer()); + await room.append({ upload: id, offset, bytes }, context); + } + await room.share({ upload: id, text: "" }, context); +} + +function Message({ channel, message }: { channel: string; message: v1.Message }) { + const { file } = message; + const url = file === undefined ? "" : `/files/${channel}/${file.id}`; + const body = + file === undefined + ? null + : file.mime.startsWith("image/") + ? h("img", { src: url, alt: file.name }) + : h("a", { href: url }, `${file.name} (${file.size} bytes)`); + return h("li", null, h("b", null, message.author), " ", message.text, body); +} + +function RoomView({ channel }: { channel: string }) { + const room = useHold(Room, { id: channel }); + const messages = useWatch(room?.messages) ?? []; + const [text, setText] = useState(""); + const send = (event: { preventDefault(): void }) => { + event.preventDefault(); + if (room === undefined || text === "") return; + void room.post(text, context); + setText(""); + }; + return h( + "section", + null, + h("ul", null, ...messages.map((message) => h(Message, { key: message.id, channel, message }))), + h( + "form", + { onSubmit: send }, + h("input", { value: text, onChange: (event: { target: { value: string } }) => setText(event.target.value) }), + h("input", { + type: "file", + onChange: (event: { target: { files: FileList | null } }) => { + const file = event.target.files?.[0]; + if (room !== undefined && file !== undefined) void upload(room, file); + }, + }), + ), + ); +} + +function Chat() { + const channels = useWatch(useHold(Channels, {})?.names) ?? []; + const [channel, setChannel] = useState("general"); + return h( + "main", + null, + h( + "nav", + null, + ...channels.map((name) => + h( + "button", + { key: name, type: "button", onClick: () => setChannel(name), "aria-pressed": name === channel }, + `#${name}`, + ), + ), + ), + h(RoomView, { key: channel, channel }), + ); +} + +void startPage({ + connect: () => new WebSocket(`${location.origin.replace(/^http/, "ws")}/chord`), + modules: { "@chat/app/v1": v1, react: React, "react/jsx-runtime": jsx }, + render: (host) => createRoot(document.getElementById("root")!).render(h(HostProvider, { host }, h(Chat))), +}); diff --git a/packages/chord/test/next/examples/59-chat/web/style.css b/packages/chord/test/next/examples/59-chat/web/style.css new file mode 100644 index 000000000..ccfea4873 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/web/style.css @@ -0,0 +1,7 @@ +/* The chat's look: the active theme's colours (the kit writes them on :root). */ +body { margin: 0; font: 15px system-ui, sans-serif; background: var(--bg); color: var(--fg); } +main { max-width: 40rem; margin: 1rem auto; } +nav button[aria-pressed="true"] { color: var(--accent); } +ul { list-style: none; padding: 0; } +li img { display: block; max-width: 16rem; margin-top: 0.25rem; } +a { color: var(--accent); } diff --git a/packages/chord/test/next/examples/59-chat/worker.ts b/packages/chord/test/next/examples/59-chat/worker.ts new file mode 100644 index 000000000..982eefcd2 --- /dev/null +++ b/packages/chord/test/next/examples/59-chat/worker.ts @@ -0,0 +1,6 @@ +// A channel's worker: the kit's `startWorker` with the chat's API. + +import { startWorker } from "../../../../src/next-node/app/index.ts"; +import { API, KINDS } from "./start.ts"; + +startWorker({ api: API, kinds: KINDS, onError: (error) => console.error("channel:", error) }); diff --git a/packages/chord/test/next/link/values.test.ts b/packages/chord/test/next/link/values.test.ts index 84656b516..a92b5562d 100644 --- a/packages/chord/test/next/link/values.test.ts +++ b/packages/chord/test/next/link/values.test.ts @@ -29,6 +29,14 @@ describe("payloads", () => { expect(read.thumbs).toEqual(thumbs); }); + it("carry any Uint8Array subclass as bytes: a Node Buffer, whose toJSON the replacer never sees (slice 11)", () => { + const buffer = Buffer.from([1, 2, 3]); + const { json, segments } = payloadPart({ bytes: buffer, at: [buffer] }, true); + expect(json).toBe('{"bytes":{"$bytes":0},"at":[{"$bytes":1}]}'); + expect(segments![0]).toBe(buffer); + expect(payloadPart(buffer, true)).toEqual({ json: '{"$bytes":0}', segments: [buffer] }); + }); + it("marked as carrying bytes refuse the reserved key $bytes with invalid/args", () => { let thrown: unknown; try { diff --git a/packages/chord/test/next/surfaces/active-theme.test.ts b/packages/chord/test/next/surfaces/active-theme.test.ts new file mode 100644 index 000000000..540306f98 --- /dev/null +++ b/packages/chord/test/next/surfaces/active-theme.test.ts @@ -0,0 +1,123 @@ +// Tests of the active theme of a client (slice 11 §4b) and of theme choices on the server: a client follows the +// `Themes` registry, its principal's `ThemeChoice` and its mode; choices are kept per principal across restarts. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + createHost, + defineFacet, + followActiveTheme, + hostKind, + memoryLinkPair, + openStore, + sqliteBackend, + type Theme, + ThemeChoice, + Themes, + themeChoiceFacet, + themesFacet, +} from "../../../src/next/index.ts"; +import { openNodeSqliteDatabase } from "../../../src/sqlite/node.ts"; + +type TestPrincipal = { readonly kind: "server" } | { readonly kind: "user"; readonly id: string }; + +const COLORS = { + accent: "#111111", + bg: "#000000", + fg: "#ffffff", + muted: "#888888", + error: "#f00", + warning: "#ff0", + success: "#0f0", +}; + +const theme = (id: string, mode: Theme["mode"]): Theme => ({ id, name: id, mode, colors: COLORS }); + +/** A server with theme choices, and a client of `user` with themes `themes` contributed by a plugin. */ +async function setup(user: string, themes: readonly Theme[]) { + const store = await openStore(sqliteBackend(await openNodeSqliteDatabase(":memory:")), context); + const server = createHost({ + address: hostKind("server")(), + principal: { kind: "server" }, + store, + onError: () => {}, + }); + await server.load([{ name: "chord.themes.choice", facet: themeChoiceFacet() }], context); + const client = createHost({ onError: () => {} }); + await client.load( + [ + { name: "chord.themes", facet: themesFacet() }, + ...themes.map((each, index) => ({ + name: `@test/theme-${index}`, + facet: defineFacet((_host, self) => self.contribute(Themes, each)), + })), + ], + context, + ); + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user", id: user } }, context); + client.connect(mine, { principal: { kind: "server" } }, context); + return { server, client }; +} + +/** Waits until `check` holds. */ +async function until(check: () => boolean): Promise { + for (let index = 0; index < 200 && !check(); index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(check()).toBe(true); +} + +describe("following the active theme", () => { + it("follows the registry, the principal's pick and the mode", async () => { + const { server, client } = await setup("ada", [ + theme("night", "dark"), + theme("dusk", "dark"), + theme("paper", "light"), + ]); + const seen: string[] = []; + const follower = await followActiveTheme(client, "dark", (active) => seen.push(active.id), context); + await until(() => seen.at(-1) === "night"); + const choice = await client.get(ThemeChoice, {}, context); + await choice.choose({ dark: "dusk", light: null }, context); + await until(() => seen.at(-1) === "dusk"); + follower.setMode("light"); + expect(seen.at(-1)).toBe("paper"); + follower.stop(); + choice.release(); + await client.close(context); + await server.close(context); + }); +}); + +describe("theme choices", () => { + it("refuse an id over 128 characters (Opus M4, Fable L7)", async () => { + const { server, client } = await setup("ada", []); + const choice = await client.get(ThemeChoice, {}, context); + await expect(choice.choose({ dark: "x".repeat(129), light: null }, context)).rejects.toMatchObject({ + code: "invalid", + }); + await choice.choose({ dark: "x".repeat(128), light: null }, context); + choice.release(); + await client.close(context); + await server.close(context); + }); + + it("are kept per principal", async () => { + const { server, client } = await setup("ada", []); + const choice = await client.get(ThemeChoice, {}, context); + expect(choice.pick.value).toEqual({ dark: null, light: null }); + await choice.choose({ dark: "dusk", light: "paper" }, context); + await until(() => choice.pick.value.dark === "dusk"); + const other = createHost({ onError: () => {} }); + await other.load([], context); + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user", id: "grace" } }, context); + other.connect(mine, { principal: { kind: "server" } }, context); + const graces = await other.get(ThemeChoice, {}, context); + expect(graces.pick.value).toEqual({ dark: null, light: null }); + choice.release(); + graces.release(); + await other.close(context); + await client.close(context); + await server.close(context); + }); +}); diff --git a/packages/chord/test/next/surfaces/client.test.ts b/packages/chord/test/next/surfaces/client.test.ts new file mode 100644 index 000000000..82017c549 --- /dev/null +++ b/packages/chord/test/next/surfaces/client.test.ts @@ -0,0 +1,179 @@ +// Tests of a client host with its plugins (slice 11 §2, §4, decisions "Slice 11, P4" to "P6"), over an in-memory link +// to a server serving the scripts object: the initial scripts loaded with the core facets, a new set applied in place +// (only changed scripts fetched; a failed one keeps its old facet), a new base or an outdated server ending the client. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { + constant, + createHost, + defineFacet, + hostKind, + local, + memoryLinkPair, + type RegisteredScript, + type ScriptSet, + type SocketEvents, + startClient, + str, + WebScripts, +} from "../../../src/next/index.ts"; +import { publishTo, scriptsFacet } from "../../../src/next-node/surfaces/current.ts"; + +type TestPrincipal = { readonly kind: "server" } | { readonly kind: "user"; readonly id: string }; + +const Greeting = local("test.greeting", { version: 1, params: {}, members: { text: str() } }); + +/** A registered script of plugin `plugin` (digest `digest`) serving `Greeting` with `text`; `broken` throws instead. */ +function script(plugin: string, digest: string, text: string): RegisteredScript { + return { + plugin, + digest, + run: (require, module) => { + if (text === "broken") throw new Error("broken plugin"); + const chord = require("@test/chord") as typeof chordNext; + const facet = chord.defineFacet((host) => host.serve(Greeting, () => ({ text: chord.constant(text) }))); + Object.assign(module, { exports: { default: facet } }); + }, + }; +} + +/** The set of `scripts` under base `base`. */ +function setOf(base: string, scripts: readonly RegisteredScript[]): ScriptSet { + return { + base, + combined: "/combined.js", + plugins: scripts.map(({ plugin, digest }) => ({ name: plugin, digest, url: `/${plugin}.${digest}.js` })), + }; +} + +/** A server serving `WebScripts` with `initial`, and a client started against it with `scripts` available to fetch. */ +async function setup(initial: readonly RegisteredScript[], available: readonly RegisteredScript[] = []) { + const server = createHost({ + address: hostKind("server")(), + principal: { kind: "server" }, + onError: () => {}, + }); + await server.load( + [{ name: "chord.web", facet: scriptsFacet("web", { set: setOf("B1", initial) }) }], + context, + ); + const fetched: string[] = []; + const errors: chordNext.ClientError[] = []; + const outdated: string[] = []; + const sockets: { events: SocketEvents; link: chordNext.Link }[] = []; + let serverBase = "B1"; + const client = await startClient({ + scripts: WebScripts, + base: "B1", + initial, + modules: { "@test/chord": chordNext }, + core: [ + { name: "@test/core", facet: defineFacet((host) => host.serve(Echo, () => ({ text: constant("core") }))) }, + ], + open: (events) => { + const [mine, theirs] = memoryLinkPair(); + server.accept(theirs, { principal: { kind: "user", id: "u" } }, context); + sockets.push({ events, link: mine }); + queueMicrotask(() => events.opened()); + return mine; + }, + fetchScript: async (entry) => { + fetched.push(entry.url); + const found = available.find(({ plugin, digest }) => plugin === entry.name && digest === entry.digest); + if (found === undefined) throw new Error(`no ${entry.url}`); + return found; + }, + serverBase: async () => serverBase, + outdated: () => outdated.push("outdated"), + environment: { + now: () => Date.now(), + after: (ms, callback) => { + const timer = setTimeout(callback, ms); + return () => clearTimeout(timer); + }, + online: () => true, + onWake: () => () => {}, + }, + onError: (error) => errors.push(error), + }); + return { + server, + client, + fetched, + errors, + outdated, + sockets, + setServerBase: (base: string) => { + serverBase = base; + }, + }; +} + +const Echo = local("test.echo", { version: 1, params: {}, members: { text: str() } }); + +/** Waits until `check` holds. */ +async function until(check: () => boolean): Promise { + for (let index = 0; index < 200 && !check(); index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(check()).toBe(true); +} + +describe("loading", () => { + it("loads the initial scripts with the core facets", async () => { + const { client, fetched, server } = await setup([script("@test/a", "d1", "hello")]); + expect((await client.host.get(Greeting, {}, context)).text.value).toBe("hello"); + expect((await client.host.get(Echo, {}, context)).text.value).toBe("core"); + expect(fetched).toEqual([]); + await client.stop(context); + await server.close(context); + }); +}); + +describe("applying a new set in place", () => { + it("fetches only changed scripts, reloads the host, keeps an unchanged plugin's facet", async () => { + const a1 = script("@test/a", "d1", "hello"); + const a2 = script("@test/a", "d2", "hello again"); + const { client, fetched, server } = await setup([a1], [a2]); + const greeting = await client.host.get(Greeting, {}, context); + await publishTo(server, "web", setOf("B1", [a2]), context); + await until(() => greeting.text.value === "hello again"); + expect(fetched).toEqual(["/@test/a.d2.js"]); + await client.stop(context); + await server.close(context); + }); + + it("keeps a plugin's old facet when its new script fails, and reports it", async () => { + const a1 = script("@test/a", "d1", "hello"); + const broken = script("@test/a", "d2", "broken"); + const { client, server, errors } = await setup([a1], [broken]); + const greeting = await client.host.get(Greeting, {}, context); + await publishTo(server, "web", setOf("B1", [broken]), context); + await until(() => errors.length > 0); + expect(errors[0]).toMatchObject({ kind: "plugins", failed: [{ plugin: "@test/a" }] }); + expect(greeting.text.value).toBe("hello"); + await client.stop(context); + await server.close(context); + }); +}); + +describe("outdated clients", () => { + it("calls outdated for a set with another base", async () => { + const { client, server, outdated } = await setup([]); + await publishTo(server, "web", setOf("B2", []), context); + await until(() => outdated.length === 1); + await client.stop(context); + await server.close(context); + }); + + it("asks the server's base before reconnecting: another one calls outdated and stops", async () => { + const { client, server, outdated, sockets, setServerBase } = await setup([]); + setServerBase("B2"); + sockets[0]!.link.close(); + await until(() => outdated.length === 1); + await new Promise((resolve) => setTimeout(resolve, 400)); + expect(sockets).toHaveLength(1); + await client.stop(context); + await server.close(context); + }); +}); diff --git a/packages/chord/test/next/surfaces/connection.test.ts b/packages/chord/test/next/surfaces/connection.test.ts new file mode 100644 index 000000000..68171f865 --- /dev/null +++ b/packages/chord/test/next/surfaces/connection.test.ts @@ -0,0 +1,214 @@ +// Tests of the connection loop (slice 11 §2, decision "Slice 11, P6"): reconnect pauses (250 ms doubling to 5 s, reset +// by a stable connection, the longest while offline), wakes (a pending retry at once; an open socket silent past the +// server's pings replaced), the connect timeout, `canRetry` stopping it, and the states it reports. + +import { describe, expect, it } from "vitest"; +import { + CONNECT_TIMEOUT_MS, + type ConnectionState, + keepConnected, + type LoopEnvironment, + type SocketEvents, + STABLE_MS, + STALE_MS, +} from "../../../src/next/surfaces/connection.ts"; + +/** A clock and timers the test advances, wakes it sends, and the sockets the loop opened. */ +class FakeWorld { + now = 0; + online = true; + readonly #timers: { at: number; callback: () => void; cancelled: boolean }[] = []; + #wake: (() => void) | undefined; + readonly sockets: FakeSocket[] = []; + readonly opens: number[] = []; + + readonly environment: LoopEnvironment = { + now: () => this.now, + after: (ms, callback) => { + const timer = { at: this.now + ms, callback, cancelled: false }; + this.#timers.push(timer); + return () => { + timer.cancelled = true; + }; + }, + online: () => this.online, + onWake: (listener) => { + this.#wake = listener; + return () => { + this.#wake = undefined; + }; + }, + }; + + open = (events: SocketEvents): FakeSocket => { + const socket = new FakeSocket(events); + this.sockets.push(socket); + this.opens.push(this.now); + return socket; + }; + + /** Advances the clock by `ms`, running due timers in order; settles promises after each. */ + async advance(ms: number): Promise { + const end = this.now + ms; + for (;;) { + await settled(); + const due = this.#timers.filter(({ at, cancelled }) => !cancelled && at <= end).sort((a, b) => a.at - b.at)[0]; + if (due === undefined) break; + this.now = due.at; + due.cancelled = true; + due.callback(); + } + this.now = end; + await settled(); + } + + wake(): void { + this.#wake?.(); + } + + get last(): FakeSocket { + return this.sockets.at(-1)!; + } +} + +/** One socket and its link: the test opens, feeds and closes it. */ +class FakeSocket { + readonly events: SocketEvents; + closed = false; + #onClose: (() => void) | undefined; + + constructor(events: SocketEvents) { + this.events = events; + } + + onClose(listener: () => void): void { + this.#onClose = listener; + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.#onClose?.(); + } +} + +async function settled(): Promise { + for (let index = 0; index < 5; index++) await Promise.resolve(); +} + +/** A loop in `world` recording its states; `retry` answers `canRetry`. */ +function loop(world: FakeWorld, retry: () => Promise = async () => true) { + const states: ConnectionState[] = []; + const handle = keepConnected({ + open: world.open, + onState: (state) => states.push(state), + canRetry: retry, + environment: world.environment, + }); + return { states, handle }; +} + +describe("reconnecting", () => { + it("reconnects after a growing pause, 250 ms doubling to 5 s, and reports its states", async () => { + const world = new FakeWorld(); + const { states } = loop(world); + const pauses: number[] = []; + for (let attempt = 0; attempt < 7; attempt++) { + const closedAt = world.now; + world.last.close(); + await world.advance(5000); + pauses.push(world.opens.at(-1)! - closedAt); + } + expect(pauses).toEqual([250, 500, 1000, 2000, 4000, 5000, 5000]); + expect(states.slice(0, 4)).toEqual(["connecting", "waiting", "connecting", "waiting"]); + }); + + it("starts the pauses again after a connection that stayed open 10 s", async () => { + const world = new FakeWorld(); + const { states } = loop(world); + for (let attempt = 0; attempt < 3; attempt++) { + world.last.close(); + await world.advance(5000); + } + world.last.events.opened(); + expect(states.at(-1)).toBe("connected"); + await world.advance(STABLE_MS); + const before = world.now; + world.last.close(); + await world.advance(5000); + expect(world.opens.at(-1)! - before).toBe(250); + }); + + it("waits the longest pause while offline, and retries at once on a wake", async () => { + const world = new FakeWorld(); + loop(world); + world.online = false; + world.last.close(); + await world.advance(1000); + expect(world.sockets).toHaveLength(1); + world.wake(); + await settled(); + expect(world.sockets).toHaveLength(2); + }); + + it("gives up a socket not open after 20 s", async () => { + const world = new FakeWorld(); + loop(world); + await world.advance(CONNECT_TIMEOUT_MS - 1); + expect(world.sockets[0]!.closed).toBe(false); + await world.advance(1); + expect(world.sockets[0]!.closed).toBe(true); + }); + + it("stops when canRetry says no", async () => { + const world = new FakeWorld(); + const { states } = loop(world, async () => false); + world.last.close(); + await world.advance(60_000); + expect(world.sockets).toHaveLength(1); + expect(states.at(-1)).toBe("waiting"); + }); + + it("retries at once when woken while canRetry is deciding", async () => { + const world = new FakeWorld(); + let answer!: (retry: boolean) => void; + loop( + world, + () => + new Promise((resolve) => { + answer = resolve; + }), + ); + world.last.close(); + world.wake(); + answer(true); + await world.advance(0); + expect(world.sockets).toHaveLength(2); + }); +}); + +describe("a silent socket", () => { + it("is replaced on a wake once its next ping is overdue; one with recent frames stays", async () => { + const world = new FakeWorld(); + loop(world); + world.last.events.opened(); + await world.advance(10_000); + world.last.events.received(); + world.wake(); + await world.advance(STALE_MS); + expect(world.sockets[0]!.closed).toBe(false); + world.wake(); + await world.advance(STALE_MS + 1000); + expect(world.sockets[0]!.closed).toBe(true); + }); + + it("closes the link at stop and opens no other", async () => { + const world = new FakeWorld(); + const { handle } = loop(world); + world.last.events.opened(); + handle.stop(); + expect(world.last.closed).toBe(true); + await world.advance(60_000); + expect(world.sockets).toHaveLength(1); + }); +}); diff --git a/packages/chord/test/next/surfaces/registered.test.ts b/packages/chord/test/next/surfaces/registered.test.ts new file mode 100644 index 000000000..1ae367fcd --- /dev/null +++ b/packages/chord/test/next/surfaces/registered.test.ts @@ -0,0 +1,57 @@ +// Tests of scripts that register themselves (slice 11 §1, §2): the served text calls `chordFacet` with the plugin, the +// digest and the build's function; evaluating it answers only the client's modules (an ES module namespace as a +// CommonJS import sees it) and refuses a default export that is no facet. + +import { Script } from "node:vm"; +import { describe, expect, it } from "vitest"; +import * as chordNext from "../../../src/next/index.ts"; +import { + evaluateRegistered, + type FacetFunction, + REGISTER, + type RegisteredScript, + registeredSource, +} from "../../../src/next/surfaces/registered.ts"; + +/** Runs a registered script's text and returns what it registered. */ +function run(text: string): RegisteredScript[] { + const registered: RegisteredScript[] = []; + (globalThis as Record)[REGISTER] = (plugin: string, digest: string, f: FacetFunction) => + registered.push({ plugin, digest, run: f }); + new Script(text).runInThisContext(); + delete (globalThis as Record)[REGISTER]; + return registered; +} + +/** A built script's function expression, as esbuild wraps a facet: it requires `specifier`, exports a facet. */ +const BUILT = (specifier: string) => + `(function (require, module, exports) { +const chord = require(${JSON.stringify(specifier)}); +module.exports = { default: chord.defineFacet(() => {}) }; +//# sourceMappingURL=data:application/json;base64,e30= +})`; + +describe("registered scripts", () => { + it("register their plugin, digest and function, the source map comment kept inside the call", () => { + const text = registeredSource("@acme/notes", "abc123", BUILT("@acme/app/v1")); + expect(text.startsWith('chordFacet("@acme/notes","abc123",(function (require, module, exports) {')).toBe(true); + const [script] = run(text); + expect([script!.plugin, script!.digest]).toEqual(["@acme/notes", "abc123"]); + const facet = evaluateRegistered(script!, { "@acme/app/v1": chordNext }); + expect(typeof facet.setup).toBe("function"); + }); + + it("answer only the client's modules, naming the plugin", () => { + const [script] = run(registeredSource("@acme/notes", "d", BUILT("node:fs"))); + expect(() => evaluateRegistered(script!, { "@acme/app/v1": chordNext })).toThrow( + 'plugin "@acme/notes": "node:fs" is not one of this client\'s host packages', + ); + }); + + it("refuse a default export that is no facet", () => { + const [script] = run( + registeredSource("@acme/bad", "d", "(function (require, module) { module.exports = { default: 42 }; })"), + ); + expect(() => evaluateRegistered(script!, {})).toThrow('plugin "@acme/bad": its default export is not a facet'); + }); +}); diff --git a/packages/chord/test/next/surfaces/themes.test.ts b/packages/chord/test/next/surfaces/themes.test.ts new file mode 100644 index 000000000..2731b21c1 --- /dev/null +++ b/packages/chord/test/next/surfaces/themes.test.ts @@ -0,0 +1,70 @@ +// Tests of themes (slice 11 §4b): the active theme for a mode and a pick, CSS variables of a theme, and terminal +// colours by role (true colour, the nearest of 256, `tui` overrides first, the fallback for a missing role). + +import { describe, expect, it } from "vitest"; +import { + activeTheme, + FALLBACK_THEMES, + type Theme, + terminalColor, + themeVariables, +} from "../../../src/next/surfaces/themes.ts"; + +const COLORS = { + accent: "#e8590c", + bg: "#111317", + fg: "#e8e8e8", + muted: "#8a8f98", + error: "#ff0000", + warning: "#ffaa00", + success: "#00ff00", +}; + +const SOLARIZED: Theme = { id: "solarized", name: "Solarized", mode: "dark", colors: { ...COLORS, link: "#268bd2" } }; + +const PAPER: Theme = { id: "paper", name: "Paper", mode: "light", colors: COLORS, web: { radius: "10px" } }; + +const NIGHT: Theme = { id: "night", name: "Night", mode: "dark", colors: COLORS, tui: { accent: "208" } }; + +describe("the active theme", () => { + it("is the picked one of the mode, else the first of the mode, else the fallback", () => { + const themes = [SOLARIZED, PAPER, NIGHT]; + expect(activeTheme(themes, { dark: "night", light: null }, "dark").id).toBe("night"); + expect(activeTheme(themes, { dark: null, light: null }, "dark").id).toBe("solarized"); + expect(activeTheme(themes, { dark: "paper", light: null }, "dark").id).toBe("solarized"); + expect(activeTheme(themes, { dark: null, light: "gone" }, "light").id).toBe("paper"); + expect(activeTheme([SOLARIZED], { dark: null, light: null }, "light")).toBe(FALLBACK_THEMES.light); + }); +}); + +describe("CSS variables", () => { + it("gives every role and every web extra a custom property", () => { + expect(themeVariables(SOLARIZED)).toMatchObject({ + "--accent": "#e8590c", + "--bg": "#111317", + "--link": "#268bd2", + }); + expect(themeVariables(PAPER)["--radius"]).toBe("10px"); + }); +}); + +describe("terminal colours", () => { + it("gives true colour, or the nearest of 256, for a role", () => { + expect(terminalColor(SOLARIZED, "accent", "truecolor", "fg")).toBe("38;2;232;89;12"); + expect(terminalColor(SOLARIZED, "bg", "truecolor", "bg")).toBe("48;2;17;19;23"); + expect(terminalColor(SOLARIZED, "error", "256", "fg")).toBe("38;5;196"); + expect(terminalColor(SOLARIZED, "fg", "256", "fg")).toBe("38;5;254"); + expect(terminalColor(SOLARIZED, "bg", "256", "bg")).toBe("48;5;233"); + }); + + it("takes the theme's tui override first, the fallback for a missing role, the default for an unknown one", () => { + expect(terminalColor(NIGHT, "accent", "truecolor", "fg")).toBe("38;5;208"); + const partial = { ...SOLARIZED, colors: { ...COLORS, muted: "not a colour" } }; + expect(terminalColor(partial, "muted", "truecolor", "fg")).toBe("39"); + expect(terminalColor(SOLARIZED, "nobody", "256", "bg")).toBe("49"); + const fallback = FALLBACK_THEMES.dark.colors.accent.slice(1); + const rgb = [0, 2, 4].map((at) => Number.parseInt(fallback.slice(at, at + 2), 16)).join(";"); + const missing = { ...SOLARIZED, colors: { ...COLORS, accent: undefined as unknown as string } }; + expect(terminalColor(missing, "accent", "truecolor", "fg")).toBe(`38;2;${rgb}`); + }); +}); diff --git a/packages/chord/test/next/tui/frames.test.ts b/packages/chord/test/next/tui/frames.test.ts new file mode 100644 index 000000000..7d63caaf9 --- /dev/null +++ b/packages/chord/test/next/tui/frames.test.ts @@ -0,0 +1,26 @@ +// Tests of frames for the terminal (slice 11 §4; fix round 1, Fable L9): a write over the launcher's 64 KiB arrives +// as frames each under it, cut only between escape sequences and surrogate pairs, the filter keeping every one whole. + +import { describe, expect, it } from "vitest"; +import { filterOutput, MAX_FRAME } from "../../../src/next-node/tui/filter.ts"; +import { FRAME_UNITS, splitFrames } from "../../../src/next-tui/frames.ts"; + +describe("splitting frames", () => { + it("keeps a small write whole", () => { + expect(splitFrames("hello")).toEqual(["hello"]); + }); + + it("cuts a large true-colour paint between sequences, each frame under 64 KiB and passing the filter whole", () => { + let paint = ""; + for (let cell = 0; cell < 20_000; cell++) + paint += `\x1b[38;2;${cell % 256};12;200m${cell % 7 === 0 ? "😀" : "é"}`; + const frames = splitFrames(paint); + expect(frames.length).toBeGreaterThan(1); + expect(frames.join("")).toBe(paint); + for (const frame of frames) { + expect(frame.length).toBeLessThanOrEqual(FRAME_UNITS); + expect(Buffer.byteLength(frame)).toBeLessThanOrEqual(MAX_FRAME); + expect(filterOutput(frame)).toBe(frame); + } + }); +}); diff --git a/packages/chord/test/next/tui/scope.test.ts b/packages/chord/test/next/tui/scope.test.ts new file mode 100644 index 000000000..43375c862 --- /dev/null +++ b/packages/chord/test/next/tui/scope.test.ts @@ -0,0 +1,80 @@ +// Tests of `TuiScope` (slice 11 §4, decision "Slice 11, P9"): references held for a component, watchables and +// registries followed, one `changed` callback for every change, and everything released at `dispose`. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost, defineFacet, defineRegistry, local, method, str } from "../../../src/next/index.ts"; +import { TuiScope } from "../../../src/next-tui/scope.ts"; + +const Notes = local("test.notes", { version: 1, params: {}, members: { text: str(), write: method(str()) } }); + +const Panels = defineRegistry("test.panels", { version: 1 }); + +/** A host serving `Notes` (a writable text) and `Panels` with one contribution. */ +async function host() { + const created = createHost({ onError: () => {} }); + await created.load( + [ + { + name: "@test/notes", + facet: defineFacet((h, self) => { + h.serve(Notes, (_params, object) => { + const text = object.writable("first"); + return { + text: object.derive([text], ([value]) => value), + write: async (next) => { + text.set(next); + return null; + }, + }; + }); + h.serve(Panels, { compose: (table) => table.map(({ value }) => value) }); + self.contribute(Panels, "notes"); + }), + }, + ], + context, + ); + return created; +} + +/** Waits until `check` holds. */ +async function until(check: () => boolean): Promise { + for (let index = 0; index < 200 && !check(); index++) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(check()).toBe(true); +} + +describe("a component's scope", () => { + it("holds, follows and calls changed on every change; releases everything at dispose", async () => { + const h = await host(); + let changes = 0; + const scope = new TuiScope(h, () => changes++); + const notes = scope.hold(Notes, {}); + const text = scope.watch(notes.text); + const panels = scope.registry(Panels); + await until(() => text() === "first" && panels()?.[0] === "notes"); + const before = changes; + await notes.write("second", context); + await until(() => text() === "second" && changes > before); + scope.dispose(); + expect(notes.state.value.kind).toBe("ended"); + const after = changes; + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(changes).toBe(after); + await h.close(context); + }); + + it("reads undefined from a watchable without a value, never throwing", async () => { + const h = await host(); + const scope = new TuiScope(h, () => {}); + const read = scope.watch({ + get value(): string { + throw new Error("not ready"); + }, + watch: () => Promise.reject(new Error("not ready")), + }); + expect(read()).toBeUndefined(); + scope.dispose(); + await h.close(context); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index e8e536464..c032b98e2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,9 +13,16 @@ "@earendil-works/chord/next/plugins/runtime": ["./packages/chord/src/next-node/plugins/runtime.ts"], "@acme/app/*": ["./packages/chord/test/next/examples/49-plugin-api/app/*.ts"], "@notes/app/*": ["./packages/chord/test/next/examples/53-app/app/*.ts"], + "@board/app/*": ["./packages/chord/test/next/examples/56-web/app/*.ts"], + "@chat/app/*": ["./packages/chord/test/next/examples/59-chat/app/*.ts"], "@earendil-works/chord/next/http": ["./packages/chord/src/next-node/http/index.ts"], "@earendil-works/chord/next/workers": ["./packages/chord/src/next-node/workers/index.ts"], "@earendil-works/chord/next/app": ["./packages/chord/src/next-node/app/index.ts"], + "@earendil-works/chord/next/web": ["./packages/chord/src/next-web/index.ts"], + "@earendil-works/chord/next/web/server": ["./packages/chord/src/next-node/surfaces/index.ts"], + "@earendil-works/chord/next/react": ["./packages/chord/src/next-react/index.ts"], + "@earendil-works/chord/next/tui": ["./packages/chord/src/next-tui/index.ts"], + "@earendil-works/chord/next/tui/launcher": ["./packages/chord/src/next-node/tui/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"],