diff --git a/package-lock.json b/package-lock.json index 74451ff75..7c8fb2531 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3387,7 +3387,7 @@ "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", - "dev": true, + "devOptional": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -5823,6 +5823,14 @@ }, "engines": { "node": ">=22.19.0" + }, + "peerDependencies": { + "esbuild": "0.28.2" + }, + "peerDependenciesMeta": { + "esbuild": { + "optional": true + } } }, "packages/codemode": { diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index d088744f3..ceb4cddd1 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -16,6 +16,7 @@ - Chord v2's store (`@earendil-works/chord/next`) gains `ChordStore.commitSeries(commits, receive, ctx)`: several commits in one backend transaction, each at its own seq with its own outbox row, the store metadata written once. Each commit's check runs on a step that sees the commits before it, before its writes; a check that throws refuses that commit and stores the ones before it, while a failed statement or write rolls the whole series back, and a failed `COMMIT` stops the store. It replaces `commitInBatch`, which is removed. Durable's `StoreStorage` stores each Session commit call through it. - Chord v2 has a package export again, `@earendil-works/chord/next` (experimental): every layer's API for apps (contracts, data, documents, hosts, jobs, links, state, the store), and, marked internal, the store's surface for adapters (`StoreOwnerTables`, `OwnerTable`, the documents engine's reads and writes, the commit step, `applyBatch`). v2's store gains one id allocator for every id in it, claims per table and per document namespace, Durable's Storage semantics for a refused commit (it rejects that commit only; an unknown outcome stops the store), document histories `none`, `latest` and `rewindable` with reads at a point, archives carrying document revisions, and batch apply; v2 objects gain `self.follow` (another runtime's synchronous observers as sources) and persisted jobs (`persist: { resume }`, a stored `abort(id, ctx)`, deferred state changes). - Chord v2 (`@earendil-works/chord/next`) gains registries, writables, wake signals and root scopes (slice 8): `defineRegistry(name, { version })` tokens shared by plugins, served by the defining plugin's facet with `host.serve(R, { compose, accepts? })`; contributions from facets (`defineFacet((host, self) => self.contribute(R, value))`) and host-lifetime objects (`StoreSelf.contribute`), plain values or Chord watchables, in a table sorted by plugin name and composed lazily (once at the end of `load`'s setups, then when read or followed, in the batch that changed the table); readers through `self.registry(R)` (a declared dependency: an object activates once the registry has a value, and fails `transient` with the new reason `unavailable/undeclared` when nobody serves it) and `host.registry(R, ctx)`, a `RegistryValue` with `observe(observer, ctx)`; older contributions and readers adapted by `accepts`; the helpers `keyedEntries` and `orderedEntries`; the `onError` kinds `registry_conflict`, `registry_failed`, `contribution_failed` and `contribution_refused`. `self.writable(initial)` is an in-process value an attempt sets (identity compared); native sources (writables, registry values, `ref.state`) are derive sources. `self.retryOn(...sources)` retries a failed object when a registry value changes. `host.root(fn, context, { principal? })` runs code outside objects with a handler's `ctx`, sealed when `fn` settles, `context` aborts or the host closes. +- Chord v2 gains plugins (slice 7c): `src/next-node/plugins` (Node, to be exported as `@earendil-works/chord/next/plugins`) finds plugin directories, builds each facet with esbuild (now an optional peer dependency of `@earendil-works/chord`, imported only there), evaluates it with `vm.Script`, maps plugin stack traces (`mapStack`), installs plugin dependencies, watches plugin and base code, and rolls reloads out across hosts. `host.load` and `host.reload` work per plugin: a plugin whose setup, factory or declarations fail is left out at `load` (`{ failed }`) or kept on its old code at `reload` (`failed` in the report, `reload(facets, ctx, { failed })` for build and evaluation failures), with a closure rule keeping a consumer old when its provider was kept old; an abort after a reload's commit rejects with `details.committed`. - Chord v2, experimental and not exported yet: the next object runtime in `src/next` (contracts, hosts and objects, documents, derives, links and relays, routines and jobs, the store), with its tests and runnable examples in `test/next` and its design in `docs/next` (`design.md`, the decision log `decisions.md`, the slice plan `slices.md`). Its API changes with every slice until it gets an entry point. - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. - `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. diff --git a/packages/chord/docs/next/benchmarks.md b/packages/chord/docs/next/benchmarks.md index 54742863c..d425f4154 100644 --- a/packages/chord/docs/next/benchmarks.md +++ b/packages/chord/docs/next/benchmarks.md @@ -880,3 +880,37 @@ the gap follows the shape of the text decoder's `fromFields`, not what runs: the (19.0-19.2 µs), the one change in it that matters is the never-executed `T` (retry) case, and writing that case with an object spread again (`...(f.length >= 2 ? { hops } : …)`) restores HEAD's 19.1 µs while other edits of the same size do not. With text measured first the gap is 22.4-22.5 → 23.0-23.3 µs (+2-3%). V8 code generation, recorded as noise. + +## Slice 7c: plugins (2026-10-30) + +**The plugins component** (`test/next/bench/plugins.ts`, new): 20 plugins, one worker facet each (a store object over a +document, a derive, a method, a helper module and a copied contract; 80.9 KiB of scripts in all), Node 24.4.1. Medians +of seven runs; three runs pinned to CPU 8, one unpinned: + +| Measurement | CPU 8 | unpinned (32 CPUs) | +| --- | --- | --- | +| cold build of the 20 worker facets (one esbuild build each, all at once) | 240.1-240.7 ms | 26.4 ms | +| rebuild of one facet after an edit | 13.1-13.3 ms | 11.0 ms | +| evaluation of one facet (`vm.Script`, its source map registered) | 0.16-0.27 ms | 0.09 ms | +| reload of one plugin on a host (evaluation and `host.reload`, a pure swap) | 0.44-0.45 ms | 0.44 ms | +| heap after 100 reloads of one plugin, against after a warm-up | +230-235 KiB | +229 KiB | + +The heap does not grow per reload: 2,000, 4,000 and 8,000 reloads leave +897, +1,186 and +1,432 KiB (sublinear); the +same evaluations through plain `vm.Script` with V8's compilation cache off leave +420 KiB at 8,000 against +465 KiB +through `evaluateFacet` (V8's own per-script bookkeeping), and the kernel's reloads alone wander between +200 and ++600 KiB at any count. Old facets and their closures are collected (the GC test: 200 reloads, every facet but the last +two collected, the heap within 2 MiB of the warm-up). + +Fix round 1 (inline source maps without `sourcesContent`: positions only), CPU 8: the 20 scripts take 53.4 KiB (80.9 +KiB before); cold build 241.1 ms, rebuild 13.0 ms, evaluation 0.14 ms per facet, a reload 0.39 ms, heap after 100 +reloads +238 KiB. + +**Kernel hot paths** (per plugin at load and reload changes only staging), HEAD `18a155e2f` → build at paths of equal +length, CPU 6, three alternating runs. `calls.ts`: local call 2.69-2.71 → 2.71-2.75 µs (6,660 B both), watched +derive commit 10.34-10.36 → 10.43-10.49 µs (14,899 → 14,901 B), activation churn 58.93-59.58 → 59.44-59.73 µs +(41,452-41,470 → 41,470 B). `links.ts` binary commit to frame (median / p90): append 0.035-0.036 / 0.058-0.059 → +0.034-0.036 / 0.058-0.060 ms, changed field 0.027-0.028 / 0.031-0.033 → 0.027-0.028 / 0.031-0.033 ms, removal 0.022 / +0.026-0.028 → 0.022 / 0.026-0.027 ms; with the young generation pinned alike within 2 µs; text alike; small call +0.026-0.030 → 0.028-0.033 ms (pinned 0.030-0.035 → 0.030-0.032 ms); retained per hold equal (one binary hold 176.2 → +170.1 KiB). `reload.ts`: swap of 50 / 1,000 objects 1.44-1.73 / 30.83-32.71 → 1.52-1.53 / 30.55-33.25 ms, restart +1.01-1.05 / 15.54-16.87 → 0.95-0.99 / 15.90-16.02 ms until the reload resolves. diff --git a/packages/chord/package.json b/packages/chord/package.json index 895c16523..f00d184dd 100644 --- a/packages/chord/package.json +++ b/packages/chord/package.json @@ -93,6 +93,14 @@ "@earendil-works/pi-telemetry": "1.4.0", "typebox": "1.3.27" }, + "peerDependencies": { + "esbuild": "0.28.2" + }, + "peerDependenciesMeta": { + "esbuild": { + "optional": true + } + }, "devDependencies": { "@types/ws": "8.18.2", "shx": "0.3.4", diff --git a/packages/chord/src/next-node/plugins/README.md b/packages/chord/src/next-node/plugins/README.md new file mode 100644 index 000000000..4926c2fc4 --- /dev/null +++ b/packages/chord/src/next-node/plugins/README.md @@ -0,0 +1,95 @@ +# plugins + +The plugins component (slice 7c, design §21 "Loading plugins and the reload loop"): plugin directories found, each +facet built into a script by esbuild and evaluated on Node, loaded and reloaded per plugin, watched and rolled out. A +Chord app modifies itself (the owner or an agent edits a plugin while it runs), so this is core machinery, no dev mode. +Node-only (`fs`, esbuild, `vm`); the kernel in `src/next` stays portable and takes facets however they were made. +esbuild is an optional peer dependency of `@earendil-works/chord`, imported only here. Exported as +`@earendil-works/chord/next/plugins` once v2 has its entry point. + +## For apps (`index.ts`) + +```ts +const plugins = createPlugins({ roots: ["./plugins", join(dataDir, "plugins")], kinds: ["server", "worker", "web", "tui"], + browserKinds: ["web"], hostPackages: HOST, base: ["./src"], onError }, ctx); +await plugins.load(server, { kind: "server", core: [{ name: "@app/core", facet: coreServerFacet }] }, ctx); +plugins.state // { dirty, install, set, base }, a JSON watchable +const built = await plugins.build(plugins.state.value.dirty, ctx); +await plugins.rollout([plugins.step("server", server), ...workerSteps], built, ctx); // the app's order +``` + +- **A plugin** is a directory with a `package.json`: its `name` (npm style) is the plugin's name, `dependencies` are + bundled, `peerDependencies` are host packages, `"chord": { "external": [...] }` keeps named packages external in + Node builds (loaded from the plugin's `node_modules`, native addons included; they stay in Node's module cache until + a restart). One file per facet kind the app names (`server.ts`, `worker.ts`, `web.tsx`, `tui.ts`; `.ts`, `.tsx`, + `.js` or `.mjs`), its default export a facet; a plugin with no facet file holds contracts others import. +- `discoverPlugins(roots, { kinds }, ctx)`: lists each root one level deep, reads only `package.json`, follows + symlinks to real paths; the first root wins a duplicate name. `{ plugins, problems }`: a directory without a valid + `package.json` or name, with two files of one kind, or a later duplicate is a problem, not a plugin. +- `buildFacet(plugin, kind, { hostPackages, platform, plugins?, publicPath? }, ctx)`: one esbuild build from cold. + Inlined: the plugin's own files, files of other discovered plugins (copies, listed in `copied`; they may import only + host packages and files of their own plugin: contracts and side-effect-free modules), packages of its + `dependencies`. External: host packages, `@earendil-works/chord/next/plugins`, `chord.external` packages and Node + builtins (Node builds; a browser build refuses both). Browser builds collect CSS into one stylesheet (`css`) and + assets (`import url from "./icon.svg?url"`, `.wasm`) into content-hashed files (`assets`) under `publicPath`. + Refused, with the file and line: `import.meta`, `__dirname`, `__filename` (each use, found through the script's + source map), a literal dynamic `import("x")` and top-level `await` (pointing to `pluginFile`; a computed `import(x)` + cannot be seen at build: it runs through the facet's `require`, which names the facet when it does not answer), a + relative path out of the plugin (base code is a host package), a package outside its `dependencies`. The result is plain data: `{ kind: "script", source, digest, inputs, copied, css?, assets }` (a + wrapped CommonJS script, minified, with an inline source map of positions only, relative to the plugin, so the + digest is the same on every checkout; `inputs` hashed as esbuild read them) or `{ kind: "failed", errors }`, never a throw. +- `evaluateFacet(built, { hostPackages, plugin })`: `vm.Script` in this context under a stable filename per (plugin, + kind, digest), the default export checked to be a facet. Its `require` answers the host packages (a CommonJS + package's exports as they are, an ESM namespace wrapped once with `__esModule` so default and named imports both + work), `@earendil-works/chord/next/plugins` with `pluginFile` bound to the plugin's directory, `chord.external` + packages and Node builtins from the plugin's directory; anything else throws naming the facet. No module map, no + cache: an old facet's closures are collected once nothing holds them. +- `pluginFile(path)` (inside a facet script): the absolute path of a file of its own plugin; `..` out of it refused. +- `mapStack(error)`: the error's stack with plugin scripts' positions mapped through their source maps + (`notes/worker.ts:12:3`), for the app's reporters; nothing global. A script's map is kept while any evaluation of it + lives (its facet, on any host, or the error its evaluation threw). +- `installDependencies(plugin, ctx, { npmCommand? })`: `npm install --omit=dev --legacy-peer-deps --ignore-scripts + --no-audit --no-fund` in the plugin's real directory. +- `createPlugins(options, ctx)`: turns V8's compilation cache off (once per process) and starts the watcher. + - `load(host, { kind, core }, ctx)`: discovers (once), builds and evaluates the `kind` facet of every plugin and + calls `host.load([...core, ...facets])`; a plugin whose build or evaluation fails is left out, reported in the + report's `failed` beside the host's own. + - `build(names, ctx)`: discovers again when the plugin set or a plugin's `package.json` changed, builds every facet + of `names` and of plugins discovered since: `{ facets, plugins, removed }`, plain data (`plugins`: the built + plugins as discovered, by name; a clone reloads the same). + - `reload(host, built, ctx)`, one after another per host: the complete set for the host's kind (the plugins it + loaded and those `built` brings): core facets and unchanged plugins (same digest) as the same objects, new scripts + evaluated, the plugins `built` removed or rebuilt without a facet of the host's kind left out (its own + discovery's, never a later one's); a plugin whose build or evaluation failed is + passed as its loaded facet (an added one left out) and named to `host.reload`'s `failed` option, so the host's + closure rule sees it and its report lists it. What the host committed becomes the loaded set, also when the + reload rejects `aborted` after its commit (`details.committed`). + - `step(name, host)`: an in-process rollout step; `rollout(steps, built, ctx)` runs steps in order and stops at one + that rejects (`stopped`); failed plugins do not stop it. It clears the dirty marks of the built plugins applied on + every step and unchanged since the build; a plugin failed on any step stays dirty. Given a copy of a build (not the + object `build` returned), it clears no marks. + - `installDependencies(name, ctx)` (the install mark clears, the plugin turns dirty), `close()` (the watcher stops). +- **The watcher** (`fs.watch` on every directory of the roots, the base code and plugin directories linked from + outside the roots, never on `node_modules` or `.git`; a directory appearing later under a watched one gets its watch, + a root missing at `createPlugins` is not watched: the app builds on demand) marks, never applies: a changed file + dirties every plugin whose facets read it (a copied contract every user) and the plugin whose directory holds it; a + `package.json` sets `install` (and the next build reads the plugin's metadata again; a plugin the next discovery no + longer has loses its install mark); a directory under a root or a facet file at a plugin's top sets `set`; a file of + base code sets `base` (the app restarts its hosts). Each mark records the generation of its latest change + (generations only increase): a change during a build keeps the mark, however late an older rollout ends. A watcher + that cannot watch reports `watch_failed` once; the app builds on demand. + +## Examples + +Each a folder in `test/next/examples/` with its hosts in `main.ts` and real plugin directories in `plugins/`: +`35-plugins` (load, edit, mark, build, roll out), `36-plugin-failures` (per plugin failures at startup and reload, the +closure rule, an abort after the commit), `37-plugin-imports` (what a facet may import, installs, the refusals), +`38-watch-and-rollout` (every mark, added, removed and dropped facets, stopped rollouts) and `39-browser-build` (a +browser build: stylesheet, assets by URL, plain data). + +## Inside the directory + +`discover.ts` (discovery), `build.ts` (one build, its import rules and input capture), `evaluate.ts` (evaluation, the +`require` a script gets, `pluginFile`), `stack.ts` (the source maps of evaluated scripts while their facets live, +`mapStack`), `install.ts`, `watcher.ts` (the watcher and its marks, a JSON watchable), `plugins.ts` (the component: +per-host loaded facets, `build`, `load`, `reload`, `step`, `rollout`). diff --git a/packages/chord/src/next-node/plugins/build.ts b/packages/chord/src/next-node/plugins/build.ts new file mode 100644 index 000000000..c09cc9896 --- /dev/null +++ b/packages/chord/src/next-node/plugins/build.ts @@ -0,0 +1,372 @@ +// Building (slice 7c §2): one facet of one plugin as one script, by esbuild from cold. Its own files, files of other +// discovered plugins (copies) and its npm `dependencies` are inlined; host packages, this component, `chord.external` +// packages and Node builtins stay external, answered at evaluation by the `require` the script takes. The result is +// plain data, the same value a worker or a page evaluates where it runs; a failed build is a result, never a throw. + +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { isBuiltin } from "node:module"; +import { basename, dirname, extname, isAbsolute, relative, resolve, sep } from "node:path"; +import { type BuildFailure, build, type Plugin as EsbuildPlugin, type Loader, type Message } from "esbuild"; +import type { Context } from "../../context/index.ts"; +import type { Plugin } from "./discover.ts"; +import { inlineSourceMap } from "./stack.ts"; + +/** What `buildFacet` takes besides the plugin and the kind. */ +export type BuildOptions = { + /** The host packages' specifiers (`@earendil-works/chord/next`, `react`, `react/jsx-runtime`, `@app/core`): external. */ + readonly hostPackages: readonly string[]; + /** Where the script runs: Node (a server, a worker, a TUI) or a browser (a page). */ + readonly platform: "node" | "browser"; + /** The discovered plugins: a file of one of them may be imported (a copy). */ + readonly plugins?: readonly Plugin[]; + /** Where the app serves a browser build's assets: the URL prefix their imports evaluate to (`/` by default). */ + readonly publicPath?: string; +}; + +/** A file a build read, with the hash of the bytes it handed esbuild (the watcher's inputs). */ +export type BuildInput = { readonly file: string; readonly hash: string }; + +/** A built facet: its script and what it used, or the build's errors. Plain data. */ +export type BuiltFacet = + | { + readonly kind: "script"; + readonly plugin: string; + readonly facet: string; + /** The wrapped CommonJS script, minified, with an inline source map whose sources are relative to the plugin. */ + readonly source: string; + /** A hash of `source`: the same on every machine and checkout for the same files. */ + readonly digest: string; + /** Every plugin file it read (its own and copied ones; not packages), absolute paths, with their hashes. */ + readonly inputs: readonly BuildInput[]; + /** The files of other plugins inlined into it, absolute paths. */ + readonly copied: readonly string[]; + /** A browser build's stylesheet, when it imports CSS. */ + readonly css?: { readonly name: string; readonly text: string }; + /** A browser build's assets, by content-hashed file name. */ + readonly assets: readonly { readonly name: string; readonly bytes: Uint8Array }[]; + } + | { readonly kind: "failed"; readonly plugin: string; readonly facet: string; readonly errors: readonly string[] }; + +/** The one specifier of this component a script imports (`pluginFile`): external always, answered per facet. */ +export const PLUGINS_SPECIFIER = "@earendil-works/chord/next/plugins"; + +/** What a free `__dirname` and `__filename` become in a build: names no script has, found again by `freeGlobals`. */ +const GLOBAL_MARKERS = { __dirname: "__chord_plugin_dirname", __filename: "__chord_plugin_filename" }; + +/** + * Builds `plugin`'s facet of `kind` (one esbuild build from cold): a script wrapped as `(function (require, module, + * exports) { … })`, or the errors that refuse it (an import the rules of §2 do not admit, a syntax error, `import.meta`, + * `__dirname`, `__filename`, a literal dynamic `import("x")`, a top-level `await`), each naming its file and line. A + * computed `import(x)` becomes a call of the script's `require`. + */ +export async function buildFacet( + plugin: Plugin, + kind: string, + options: BuildOptions, + context: Context, +): Promise { + const failed = (errors: readonly string[]): BuiltFacet => ({ + kind: "failed", + plugin: plugin.name, + facet: kind, + errors, + }); + const entry = plugin.facets[kind]; + if (entry === undefined) return failed([`plugin "${plugin.name}" has no ${kind} facet`]); + context.abortSignal?.throwIfAborted(); + const rules = new ImportRules(plugin, options); + const browser = options.platform === "browser"; + let result: Awaited>; + try { + result = await build({ + entryPoints: [entry], + absWorkingDir: plugin.dir, + // Nothing is written: the directory only makes the source map's paths relative to the plugin. + outdir: plugin.dir, + entryNames: "[name]-[hash]", + assetNames: "[name]-[hash]", + publicPath: options.publicPath ?? "/", + write: false, + bundle: true, + format: "cjs", + platform: options.platform, + target: browser ? "es2022" : "node22", + minify: true, + legalComments: "none", + sourcemap: "inline", + // Positions only: a stack is mapped to files and lines, never shown their text. + sourcesContent: false, + // Ignore any tsconfig.json: TypeScript is stripped, JSX compiled with the automatic React runtime. + tsconfigRaw: "{}", + jsx: "automatic", + banner: { js: "(function (require, module, exports) {" }, + footer: { js: "})" }, + // `import.meta` is refused here; a free `__dirname` or `__filename` becomes a marker found in the output below + // (esbuild reports no `import.meta` it substituted itself). + define: GLOBAL_MARKERS, + logOverride: { "empty-import-meta": "error" }, + // A computed `import(x)` (a literal one is refused below) goes through the facet's `require`, which names the + // facet when it does not answer; `vm.Script` has no loader for a real one. + supported: { "dynamic-import": false }, + logLevel: "silent", + plugins: [rules.plugin], + }); + } catch (error) { + return failed( + ((error as BuildFailure).errors ?? [{ text: String(error) }]).map((each) => describe(plugin, each)), + ); + } + // Never written: esbuild hands the outputs back. + const files = result.outputFiles!; + const script = files.find((file) => file.path.endsWith(".js"))!; + const css = files.find((file) => file.path.endsWith(".css")); + const source = script.text; + const globals = freeGlobals(source); + if (globals.length > 0) return failed(globals); + return { + kind: "script", + plugin: plugin.name, + facet: kind, + source, + digest: createHash("sha256").update(source).digest("hex"), + inputs: rules.inputs(), + copied: rules.copied(), + ...(css === undefined ? {} : { css: { name: basename(css.path), text: css.text } }), + assets: files + .filter((file) => file !== script && file !== css) + .map((file) => ({ name: relative(plugin.dir, file.path), bytes: file.contents })), + }; +} + +/** Where a build error points for what a plugin script cannot have. */ +const PLUGIN_FILE_HINT = `a file of the plugin is read through pluginFile(path) from "${PLUGINS_SPECIFIER}"`; + +/** The error of a script using `import.meta`, `__dirname` or `__filename`. */ +const NO_MODULE_IDENTITY = `import.meta.url, __dirname and __filename are not available in a plugin script (${PLUGIN_FILE_HINT})`; + +/** + * The errors for every free `__dirname` and `__filename` in the built `source` (their markers), each at its file, line + * and column through the script's source map. + */ +function freeGlobals(source: string): string[] { + // Most scripts have none: no map parsed for them. + if (!source.includes("__chord_plugin_")) return []; + const map = inlineSourceMap(source); + if (map === undefined) return []; + const errors: string[] = []; + for (const [line, text] of source.split("\n").entries()) { + for (const match of text.matchAll(/__chord_plugin_(?:dirname|filename)/g)) { + const entry = map.findEntry(line, match.index); + if (!("originalSource" in entry)) continue; + errors.push( + `${entry.originalSource}:${entry.originalLine + 1}:${entry.originalColumn + 1}: ${NO_MODULE_IDENTITY}`, + ); + } + } + return errors; +} + +/** Where a resolved file lies, from the plugin being built: its own source, another plugin's (a copy), a package. */ +type Place = + | { readonly kind: "own" } + | { readonly kind: "copied"; readonly plugin: Plugin } + | { readonly kind: "package" } + | { readonly kind: "outside" }; + +/** What only a browser build imports: stylesheets and wasm assets. */ +const BROWSER_ONLY = new Set([".css", ".wasm"]); + +/** Marks esbuild's resolution the rules ask for themselves, so they do not judge it again. */ +const RESOLVING = Symbol("chord.plugins.resolving"); + +/** How the source files a build reads are loaded, by extension (packages load as esbuild decides). */ +const LOADERS: Readonly> = { + ".ts": "ts", + ".mts": "ts", + ".cts": "ts", + ".tsx": "tsx", + ".js": "js", + ".mjs": "js", + ".cjs": "js", + ".jsx": "jsx", + ".json": "json", + ".css": "css", + ".wasm": "file", +}; + +/** What an asset import's resolution carries to its load (the `file` loader whatever its extension). */ +const ASSET = Symbol("chord.plugins.asset"); + +/** + * The import rules of one build (§2's table) as an esbuild plugin, and what the build read: every file of a plugin + * hashed as it is handed to esbuild (the exact bytes built, not a later read racing a save), and the copied files. + */ +class ImportRules { + readonly plugin: EsbuildPlugin; + readonly #own: Plugin; + readonly #others: readonly Plugin[]; + readonly #options: BuildOptions; + readonly #hostPackages: ReadonlySet; + readonly #inputs = new Map(); + readonly #copied = new Set(); + + constructor(own: Plugin, options: BuildOptions) { + this.#own = own; + this.#others = (options.plugins ?? []).filter((each) => each.dir !== own.dir); + this.#options = options; + this.#hostPackages = new Set(options.hostPackages); + this.plugin = { + name: "chord-plugin-imports", + setup: (builder) => { + builder.onResolve({ filter: /.*/ }, async (args) => { + // The plugin's files are resolved here, the entry too: esbuild then gives them no Node module type + // (`"type": "module"`), whose interop would read a whole host package for a default import instead + // of honouring the `__esModule` an ESM namespace is wrapped with (§3). + if (args.kind === "entry-point") return { path: resolve(own.dir, args.path) }; + if (args.pluginData === RESOLVING) return undefined; + const resolved = await this.#resolve(args.path, args.importer, args.kind); + if (resolved.kind === "error") return { errors: [{ text: resolved.text }] }; + if (resolved.kind === "external") return { path: args.path, external: true }; + if (resolved.kind === "default") return undefined; + const found = await builder.resolve(resolved.path, { + kind: args.kind, + importer: args.importer, + resolveDir: args.resolveDir, + pluginData: RESOLVING, + }); + if (found.errors.length > 0) return { errors: found.errors }; + if (options.platform === "node" && BROWSER_ONLY.has(extname(found.path))) { + return { errors: [{ text: `"${args.path}": stylesheets and assets are for browser builds` }] }; + } + const checked = this.#checkFile(found.path, args.importer, args.path); + if (checked !== undefined) return { errors: [{ text: checked }] }; + return { path: found.path, pluginData: resolved.asset ? ASSET : undefined }; + }); + builder.onLoad({ filter: /.*/ }, async (args) => { + if (args.namespace !== "file") return undefined; + const asset = args.pluginData === ASSET; + const loader = asset ? "file" : LOADERS[extname(args.path)]; + if (loader === undefined || this.#place(args.path).kind === "package") return undefined; + const contents = await readFile(args.path); + this.#inputs.set(args.path, createHash("sha256").update(contents).digest("hex")); + return { contents, loader, resolveDir: dirname(args.path) }; + }); + }, + }; + } + + /** Every plugin file read, sorted by path. */ + inputs(): BuildInput[] { + return [...this.#inputs].map(([file, hash]) => ({ file, hash })).sort((a, b) => (a.file < b.file ? -1 : 1)); + } + + /** The other plugins' files inlined, sorted. */ + copied(): string[] { + return [...this.#copied].sort(); + } + + /** + * What an import of `specifier` from `importer` becomes: external, refused, esbuild's own resolution (a package of + * the plugin's `dependencies`, or anything a package imports), or a path esbuild resolves and the rules check. + */ + async #resolve( + specifier: string, + importer: string, + kind: string, + ): Promise< + | { readonly kind: "external" } + | { readonly kind: "default" } + | { readonly kind: "path"; readonly path: string; readonly asset: boolean } + | { readonly kind: "error"; readonly text: string } + > { + const browser = this.#options.platform === "browser"; + if (kind === "dynamic-import") { + return { + kind: "error", + text: `import("${specifier}"): a plugin script has no dynamic import (${PLUGIN_FILE_HINT})`, + }; + } + if (specifier === PLUGINS_SPECIFIER || this.#hostPackages.has(specifier)) return { kind: "external" }; + if (isBuiltin(specifier)) { + if (!browser) return { kind: "external" }; + return { kind: "error", text: `"${specifier}": a Node builtin in a browser build` }; + } + if (specifier.endsWith("?url")) { + if (browser) return { kind: "path", path: specifier.slice(0, -"?url".length), asset: true }; + return { kind: "error", text: `"${specifier}": assets by URL are for browser builds (${PLUGIN_FILE_HINT})` }; + } + const from = this.#place(importer); + if (from.kind === "package") return { kind: "default" }; + if (specifier.startsWith(".") || isAbsolute(specifier)) return { kind: "path", path: specifier, asset: false }; + if (from.kind === "copied") { + return { kind: "error", text: `"${specifier}": ${copiedRule(from.plugin)}` }; + } + const name = packageName(specifier); + if (this.#own.external.includes(name)) { + if (!browser) return { kind: "external" }; + return { kind: "error", text: `"${specifier}": chord.external packages are for Node builds` }; + } + if (this.#own.dependencies.includes(name)) return { kind: "default" }; + return { + kind: "error", + text: `"${specifier}": neither a host package nor in the dependencies of plugin "${this.#own.name}"`, + }; + } + + /** + * Why the file `path` (resolved from a relative import in `importer`) may not be inlined: outside the plugin and + * every other discovered plugin, or, from a copied file, outside that file's own plugin; `undefined` when it may (a + * file of another plugin is recorded as copied). + */ + #checkFile(path: string, importer: string, specifier: string): string | undefined { + const from = this.#place(importer); + const to = this.#place(path); + if (to.kind === "own" && from.kind === "own") return undefined; + if (to.kind === "copied" && (from.kind === "own" || (from.kind === "copied" && from.plugin === to.plugin))) { + this.#copied.add(path); + return undefined; + } + if (from.kind === "copied") return `"${specifier}": ${copiedRule(from.plugin)}`; + return `"${specifier}": outside the plugin (base code is reached through a host package, packages through dependencies)`; + } + + /** Where `path` lies. */ + #place(path: string): Place { + if (path.split(sep).includes("node_modules")) return { kind: "package" }; + if (inside(this.#own.dir, path)) return { kind: "own" }; + const other = this.#others.find((each) => inside(each.dir, path)); + return other === undefined ? { kind: "outside" } : { kind: "copied", plugin: other }; + } +} + +/** The rule a copied file broke. */ +function copiedRule(plugin: Plugin): string { + return `a file copied from plugin "${plugin.name}" may import only host packages and files of its own plugin`; +} + +/** Whether `path` lies inside directory `dir` (both absolute). */ +export function inside(dir: string, path: string): boolean { + const rest = relative(dir, path); + return rest !== "" && !rest.startsWith("..") && !isAbsolute(rest); +} + +/** The package a bare specifier names: `@scope/name` or `name`, without a subpath. */ +export function packageName(specifier: string): string { + const parts = specifier.split("/"); + return specifier.startsWith("@") ? parts.slice(0, 2).join("/") : parts[0]!; +} + +/** One build error as `file:line:column: text`, its file relative to the plugin, with the hint where it applies. */ +function describe(plugin: Plugin, message: Message): string { + let text = message.text; + if (message.id === "empty-import-meta") { + text = NO_MODULE_IDENTITY; + } else if (text.startsWith("Top-level await")) { + text = "a plugin script has no top-level await (a facet declares synchronously)"; + } + const location = message.location; + if (location === null || location === undefined) return text; + const file = isAbsolute(location.file) ? relative(plugin.dir, location.file) : location.file; + return `${file}:${location.line}:${location.column + 1}: ${text}`; +} diff --git a/packages/chord/src/next-node/plugins/discover.ts b/packages/chord/src/next-node/plugins/discover.ts new file mode 100644 index 000000000..bff74b858 --- /dev/null +++ b/packages/chord/src/next-node/plugins/discover.ts @@ -0,0 +1,124 @@ +// Discovery (slice 7c §1): the plugins under the app's roots, found by listing each root one level deep and reading only +// each directory's `package.json` (its `name`, its `dependencies`, its `chord.external`); its facet files are the ones +// named after the app's facet kinds. Nothing parses or runs plugin code. + +import { readdir, readFile, realpath, stat } from "node:fs/promises"; +import { join } from "node:path"; +import type { Context } from "../../context/index.ts"; +import { isPluginName } from "../../next/host/staging.ts"; + +/** A plugin on disk: its name (its `package.json` name), its real directory, its facet files and npm packages. */ +export type Plugin = { + readonly name: string; + /** Its directory's real path (symlinks followed). */ + readonly dir: string; + /** Its facet files by kind, absolute paths; none for a contracts-only plugin. */ + readonly facets: Readonly>; + /** The packages in its `dependencies`: bundled into the facets that import them. */ + readonly dependencies: readonly string[]; + /** The packages in its `chord.external`: kept external in Node builds, loaded from its `node_modules`. */ + readonly external: readonly string[]; +}; + +/** A directory under a root that is not a plugin, or a plugin left out, and why. */ +export type DiscoveryProblem = { readonly path: string; readonly error: string }; + +/** What discovery found: the plugins by name, and the directories it reported. */ +export type Discovery = { readonly plugins: readonly Plugin[]; readonly problems: readonly DiscoveryProblem[] }; + +/** What discovery takes besides the roots: the app's facet kinds (`["server", "worker", "web", "tui"]`). */ +export type DiscoveryOptions = { readonly kinds: readonly string[] }; + +/** + * Lists every root one level deep: a directory with a valid `package.json` is a plugin (a missing root is empty). A + * directory without a `package.json`, with an invalid one or an invalid name, with two files of one facet kind, or whose + * name an earlier root (or an earlier directory of the same root) has already, is reported. Plugins sorted by name. + */ +export async function discoverPlugins( + roots: readonly string[], + options: DiscoveryOptions, + context: Context, +): Promise { + const plugins = new Map(); + const problems: DiscoveryProblem[] = []; + for (const root of roots) { + for (const dir of await directoriesOf(root)) { + context.abortSignal?.throwIfAborted(); + const found = await pluginAt(dir, options.kinds); + if (found.kind === "problem") { + problems.push({ path: dir, error: found.error }); + continue; + } + const other = plugins.get(found.plugin.name); + if (other !== undefined) { + problems.push({ path: dir, error: `plugin "${found.plugin.name}" is in ${other.dir} already` }); + continue; + } + plugins.set(found.plugin.name, found.plugin); + } + } + const sorted = [...plugins.values()].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + return { plugins: sorted, problems }; +} + +/** The directories directly under `root`, by real path, in name order; none when `root` does not exist. */ +async function directoriesOf(root: string): Promise { + let names: string[]; + try { + names = (await readdir(root)).sort(); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + const dirs: string[] = []; + for (const name of names) { + // Not plugins: hidden directories and a root's own packages. + if (name.startsWith(".") || name === "node_modules") continue; + const path = join(root, name); + // `stat` follows a symlink: a linked plugin directory is listed by its real path. + const info = await stat(path).catch(() => undefined); + if (info?.isDirectory() === true) dirs.push(await realpath(path)); + } + return dirs; +} + +/** The facet files' extensions, in no order of preference: two files of one kind are reported. */ +export const FACET_EXTENSIONS: readonly string[] = [".ts", ".tsx", ".js", ".mjs"]; + +/** The plugin in `dir` (its real path), or why it is none. */ +async function pluginAt( + dir: string, + kinds: readonly string[], +): Promise< + { readonly kind: "plugin"; readonly plugin: Plugin } | { readonly kind: "problem"; readonly error: string } +> { + let manifest: unknown; + try { + manifest = JSON.parse(await readFile(join(dir, "package.json"), "utf8")); + } catch (error) { + const missing = (error as NodeJS.ErrnoException).code === "ENOENT"; + return { kind: "problem", error: missing ? "no package.json" : `package.json: ${(error as Error).message}` }; + } + // A trust boundary: a plugin's own `package.json`. + const { name, dependencies, chord } = (manifest ?? {}) as { + readonly name?: unknown; + readonly dependencies?: unknown; + readonly chord?: { readonly external?: unknown }; + }; + if (typeof name !== "string" || !isPluginName(name)) { + return { kind: "problem", error: `package.json: ${JSON.stringify(name)} is not a valid plugin name` }; + } + const external = chord?.external ?? []; + if (!Array.isArray(external) || !external.every((each) => typeof each === "string")) { + return { kind: "problem", error: "package.json: chord.external is not a list of package names" }; + } + const files = new Set(await readdir(dir)); + const facets: Record = {}; + for (const kind of kinds) { + const named = FACET_EXTENSIONS.map((extension) => `${kind}${extension}`).filter((file) => files.has(file)); + if (named.length > 1) return { kind: "problem", error: `two ${kind} facets: ${named.join(", ")}` }; + if (named.length === 1) facets[kind] = join(dir, named[0]!); + } + const packages = typeof dependencies === "object" && dependencies !== null ? Object.keys(dependencies) : []; + return { kind: "plugin", plugin: { name, dir, facets, dependencies: packages, external } }; +} diff --git a/packages/chord/src/next-node/plugins/evaluate.ts b/packages/chord/src/next-node/plugins/evaluate.ts new file mode 100644 index 000000000..2b900e512 --- /dev/null +++ b/packages/chord/src/next-node/plugins/evaluate.ts @@ -0,0 +1,116 @@ +// Evaluating (slice 7c §3): a built script run with `vm.Script` in this context under a stable filename per (plugin, +// kind, digest), given the `require` that answers its external imports: the host packages (one instance per process), +// this component's `pluginFile` bound to the plugin's directory, and `chord.external` packages and Node builtins from +// the plugin's own `node_modules`. No module map and no cache: an old facet's closures are collected once nothing holds +// them. + +import { createRequire, isBuiltin } from "node:module"; +import { basename, isAbsolute, join, relative, resolve } from "node:path"; +import { Script } from "node:vm"; +import type { Principal } from "../../next/data/index.ts"; +import type { Facet } from "../../next/host/index.ts"; +import { type BuiltFacet, PLUGINS_SPECIFIER, packageName } from "./build.ts"; +import type { Plugin } from "./discover.ts"; +import { registerScript } from "./stack.ts"; + +/** + * The host packages by specifier, each what Node's `require` would give: a CommonJS package's exports, an ESM + * package's namespace (`import * as`), wrapped once so default and named imports both work. + */ +export type HostPackages = Readonly>; + +/** What `evaluateFacet` takes besides the script: the host packages and the plugin it was built from. */ +export type EvaluateOptions = { readonly hostPackages: HostPackages; readonly plugin: Plugin }; + +/** A built script, as evaluation takes it. */ +export type BuiltScript = Extract; + +/** + * Evaluates `built` and returns its facet (its default export). Throws what the script throws while it is evaluated, + * an import its `require` does not answer, or a declaration error when its default export is not a facet. + */ +export function evaluateFacet

(built: BuiltScript, options: EvaluateOptions): Facet

{ + const { plugin, hostPackages } = options; + const where = `plugin "${plugin.name}" (${built.facet})`; + const file = plugin.facets[built.facet] ?? `${built.facet}.js`; + const filename = `/chord-plugins/${plugin.name}/${basename(file)}?${built.digest}`; + // The script is one function expression: its value when run. + const wrapper = new Script(built.source, { filename }).runInThisContext() as ( + require: (specifier: string) => unknown, + module: { exports: unknown }, + exports: unknown, + ) => void; + const module = { exports: {} as unknown }; + let facet: unknown; + try { + wrapper(requireFor(plugin, hostPackages, where), module, module.exports); + // A trust boundary: plugin code; its default export is checked to be a facet (a getter of it may throw too). + facet = (module.exports as { readonly default?: unknown } | null)?.default; + if (typeof facet !== "object" || facet === null || typeof (facet as { setup?: unknown }).setup !== "function") { + throw new Error(`${where}: its default export is not a facet (defineFacet)`); + } + } catch (error) { + // Its stack maps while the error lives (an import-time throw names the plugin's line). + if (typeof error === "object" && error !== null) registerScript(filename, built.source, plugin.dir, error); + throw error; + } + registerScript(filename, built.source, plugin.dir, facet); + return facet as Facet

; +} + +/** + * Inside a facet script, the absolute path of `path`, a file of the facet's own plugin (`..` out of its directory + * refused); evaluation gives each facet its own. Imported anywhere else, it throws. + */ +export function pluginFile(path: string): string { + throw new Error(`pluginFile("${path}"): only a plugin's facet script, evaluated by the plugins component, has one`); +} + +/** The `require` a facet script of `plugin` takes (`where` names the facet in its errors). */ +function requireFor(plugin: Plugin, hostPackages: HostPackages, where: string): (specifier: string) => unknown { + let fromPlugin: NodeJS.Require | undefined; + let plugins: { readonly pluginFile: (path: string) => string } | undefined; + return (specifier) => { + if (Object.hasOwn(hostPackages, specifier)) return interop(hostPackages[specifier]); + if (specifier === PLUGINS_SPECIFIER) { + plugins ??= { pluginFile: (path) => fileOf(plugin, path) }; + return plugins; + } + if (isBuiltin(specifier) || plugin.external.includes(packageName(specifier))) { + fromPlugin ??= createRequire(join(plugin.dir, "package.json")); + return fromPlugin(specifier); + } + throw new Error( + `${where}: "${specifier}" is not a host package, a Node builtin or one of its chord.external packages`, + ); + }; +} + +/** The absolute path of `path` inside `plugin`'s directory; one out of it is refused. */ +function fileOf(plugin: Plugin, path: string): string { + const file = resolve(plugin.dir, path); + const rest = relative(plugin.dir, file); + if (rest.startsWith("..") || isAbsolute(rest)) { + throw new Error(`pluginFile("${path}"): outside the directory of plugin "${plugin.name}"`); + } + return file; +} + +/** ESM namespaces wrapped for a CommonJS script, by namespace: one wrapper per namespace, held as long as it is. */ +const wrapped = new WeakMap(); + +/** + * A host package as a CommonJS script imports it: an ESM namespace wrapped once with `__esModule` (so esbuild's interop + * reads its `default` for a default import, its names for named ones), any other value as it is. + */ +function interop(value: unknown): unknown { + if (typeof value !== "object" || value === null || Object.prototype.toString.call(value) !== "[object Module]") { + return value; + } + let made = wrapped.get(value); + if (made === undefined) { + made = { ...value, __esModule: true }; + wrapped.set(value, made); + } + return made; +} diff --git a/packages/chord/src/next-node/plugins/index.ts b/packages/chord/src/next-node/plugins/index.ts new file mode 100644 index 000000000..8445aa57b --- /dev/null +++ b/packages/chord/src/next-node/plugins/index.ts @@ -0,0 +1,25 @@ +// Chord v2's plugins component (`@earendil-works/chord/next/plugins` once v2 has its entry point; slice 7c): plugin +// directories found, built into scripts, evaluated, loaded and reloaded per plugin, watched and rolled out. Node-only +// (`fs`, esbuild, `vm`); esbuild is an optional peer dependency, imported only here. + +export { type BuildInput, type BuildOptions, type BuiltFacet, buildFacet } from "./build.ts"; +export { + type Discovery, + type DiscoveryOptions, + type DiscoveryProblem, + discoverPlugins, + type Plugin, +} from "./discover.ts"; +export { type BuiltScript, type EvaluateOptions, evaluateFacet, type HostPackages, pluginFile } from "./evaluate.ts"; +export { type InstallOptions, installDependencies } from "./install.ts"; +export { + type Built, + createPlugins, + type Plugins, + type PluginsError, + type PluginsOptions, + type RolloutResult, + type RolloutStep, +} from "./plugins.ts"; +export { mapStack } from "./stack.ts"; +export type { PluginsState } from "./watcher.ts"; diff --git a/packages/chord/src/next-node/plugins/install.ts b/packages/chord/src/next-node/plugins/install.ts new file mode 100644 index 000000000..a9b612fcb --- /dev/null +++ b/packages/chord/src/next-node/plugins/install.ts @@ -0,0 +1,47 @@ +// Installing a plugin's npm dependencies (slice 7c §5, decision "Slice 7c's points" (3)): as coding-agent installs a +// package's dependencies, `npm install` in the plugin's real directory without dev dependencies, host peers or +// lifecycle scripts; the command overridable. + +import { spawn } from "node:child_process"; +import type { Context } from "../../context/index.ts"; +import type { Plugin } from "./discover.ts"; + +/** What `installDependencies` takes besides the plugin: the npm command (`["npm"]` by default, e.g. `["pnpm"]`). */ +export type InstallOptions = { readonly npmCommand?: readonly string[] }; + +/** The install's arguments: production dependencies only, no host peers, no scripts, no audit or funding notes. */ +const INSTALL_ARGS = ["install", "--omit=dev", "--legacy-peer-deps", "--ignore-scripts", "--no-audit", "--no-fund"]; + +/** + * Installs `plugin`'s `dependencies` into its own `node_modules` (`npm install --omit=dev --legacy-peer-deps + * --ignore-scripts --no-audit --no-fund` in its real directory). Rejects with the command's output when it fails, and + * when `context` aborts (the command is killed). + */ +export async function installDependencies( + plugin: Plugin, + context: Context, + options: InstallOptions = {}, +): Promise { + const [command, ...prefix] = options.npmCommand ?? ["npm"]; + if (command === undefined) throw new Error("installDependencies: npmCommand is empty"); + context.abortSignal?.throwIfAborted(); + const child = spawn(command, [...prefix, ...INSTALL_ARGS], { + cwd: plugin.dir, + stdio: ["ignore", "pipe", "pipe"], + signal: context.abortSignal, + }); + let output = ""; + child.stdout.on("data", (chunk: Buffer) => { + output += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + output += chunk.toString(); + }); + const code = await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", resolve); + }); + if (code !== 0) { + throw new Error(`plugin "${plugin.name}": ${command} install failed (exit ${code}): ${output.trim()}`); + } +} diff --git a/packages/chord/src/next-node/plugins/plugins.ts b/packages/chord/src/next-node/plugins/plugins.ts new file mode 100644 index 000000000..c1adf3a9e --- /dev/null +++ b/packages/chord/src/next-node/plugins/plugins.ts @@ -0,0 +1,407 @@ +// The plugins component (slice 7c §4, §5; decision "Slice 7c's points"): discovery, builds, evaluation and the watcher +// together, and per host the facet loaded for every plugin, so a reload passes the host its complete plugin set +// (unchanged plugins as the same facet objects, removed ones left out) and never restarts an unchanged plugin; per +// plugin, a failed build or evaluation keeps the plugin's loaded facet (left out at startup) and is reported with the +// host's report. `rollout` runs the app's steps in order and clears the marks of plugins applied on every step. + +import { setFlagsFromString } from "node:v8"; +import type { Context } from "../../context/index.ts"; +import { isChordError, type Principal } from "../../next/data/index.ts"; +import type { + CommittedReload, + Facet, + Host, + LoadReport, + PluginFacet, + PluginFailure, + ReloadReport, +} from "../../next/host/index.ts"; +import { Batch, type BatchError } from "../../next/state/internal.ts"; +import type { Watchable } from "../../watch.ts"; +import { type BuiltFacet, buildFacet } from "./build.ts"; +import { type Discovery, discoverPlugins, type Plugin } from "./discover.ts"; +import { evaluateFacet, type HostPackages } from "./evaluate.ts"; +import { type InstallOptions, installDependencies } from "./install.ts"; +import { type MarkSnapshot, type PluginsState, PluginWatcher } from "./watcher.ts"; + +/** What `createPlugins` takes. */ +export type PluginsOptions = { + /** The directories plugins are listed in, in order: the first root's plugin wins a duplicate name. */ + readonly roots: readonly string[]; + /** The app's facet kinds (`["server", "worker", "web", "tui"]`): one file per kind in a plugin. */ + readonly kinds: readonly string[]; + /** The kinds built for a browser (`["web"]`); the others are built for Node. */ + readonly browserKinds?: readonly string[]; + /** The host packages by specifier: external in every build, given to every evaluation on Node. */ + readonly hostPackages: HostPackages; + /** The app's base code: a change sets `base` (the app restarts its hosts; Chord restarts nothing). */ + readonly base?: readonly string[]; + /** Where the app serves browser builds' assets: the URL prefix their imports evaluate to. */ + readonly publicPath?: string; + /** How `installDependencies` runs npm. */ + readonly install?: InstallOptions; + /** The watcher cannot watch (reported once), and the marks' watch listeners that threw. */ + readonly onError: (error: PluginsError) => void; +}; + +/** What the component reports. */ +export type PluginsError = BatchError | { readonly kind: "watch_failed"; readonly error: unknown }; + +/** + * A build of some plugins, plain data (a clone of it reloads the same): every facet of each, the plugins as its + * discovery saw them by name (what a reload evaluates with), and the plugins no longer discovered. + */ +export type Built = { + readonly facets: readonly BuiltFacet[]; + readonly plugins: Readonly>; + readonly removed: readonly string[]; +}; + +/** What a rollout step does: reloads its host (or hosts) with the facets of its kinds from a build. */ +export type RolloutStep = { + readonly name: string; + readonly kinds: readonly string[]; + reload(built: Built, context: Context): Promise; +}; + +/** What a rollout did: each step's report in order, and the step it stopped at (rejected), if it stopped. */ +export type RolloutResult = { + readonly steps: readonly { readonly name: string; readonly report: ReloadReport }[]; + readonly stopped: { readonly step: string; readonly error: unknown } | undefined; +}; + +/** The plugins component. */ +export interface Plugins { + /** The marks: `{ dirty, install, set, base }`, a JSON watchable. */ + readonly state: Watchable; + /** The plugins of the last discovery. */ + readonly discovery: Discovery; + /** + * Builds every facet of `names` (and of plugins discovered since the last build; discovery runs again first when + * the plugin set changed): plain data, reused by later loads and reloads. + */ + build(names: readonly string[], context: Context): Promise; + /** + * Builds (if needed) and evaluates the `kind` facet of every discovered plugin and loads them with `core` (the app's + * own facets): a plugin whose build or evaluation fails is left out and reported with the host's report. + */ + load

( + host: Host

, + options: { readonly kind: string; readonly core: readonly PluginFacet

[] }, + context: Context, + ): Promise; + /** + * Reloads `host` (loaded by `load`) with the facets of its kind in `built`, after its earlier reloads: the complete + * set (the plugins it loaded and those `built` brings), its core facets and unchanged plugins as the same objects, + * new scripts evaluated, a plugin whose build or evaluation failed passed as its loaded facet (an added one left out, + * both given to the host's closure rule), the plugins `built` removed left out. The host's report, those failures + * included; what the host committed is the loaded set also when the reload rejects `aborted` after its commit. + */ + reload

(host: Host

, built: Built, context: Context): Promise; + /** An in-process rollout step named `name`: `reload(host, built)`, for the kind `host` was loaded with. */ + step

(name: string, host: Host

): RolloutStep; + /** + * Runs `steps` in order with `built` and stops at one that rejects (a host closed, an abort); failed plugins do not + * stop it. Clears the dirty marks of the built plugins applied on every step (not failed on any) that did not + * change since the build. + */ + rollout(steps: readonly RolloutStep[], built: Built, context: Context): Promise; + /** Installs plugin `name`'s dependencies (`installDependencies`): its install mark clears, and it is dirty. */ + installDependencies(name: string, context: Context): Promise; + /** Stops the watcher. */ + close(): void; +} + +/** + * Creates the plugins component: turns V8's compilation cache off (once per process: each distinct evaluated script + * would stay cached) and starts the watcher. Discovery runs at the first `build` or `load`. + */ +export function createPlugins(options: PluginsOptions, _context: Context): Plugins { + if (!cacheOff) { + setFlagsFromString("--no-compilation-cache"); + cacheOff = true; + } + return new PluginsComponent(options); +} + +/** Whether this process turned V8's compilation cache off already. */ +let cacheOff = false; + +/** A plugin's facet as a host loaded it, with its script's digest. */ +type LoadedFacet = { readonly facet: Facet; readonly digest: string }; + +/** + * A host's plugins as the component loaded them: its kind, its core facets, each plugin's loaded facet, and its + * reloads, one after another (each computes its complete set once the previous one adopted its result). + */ +type HostPlugins = { + readonly kind: string; + readonly core: readonly PluginFacet[]; + readonly loaded: Map; + reloading: Promise; +}; + +/** + * Adopts as `state`'s loaded facets the set `next` a reload committed: a plugin the host kept old (`failed`) stays loaded + * as it was, an added one it left out is not loaded. + */ +function adopt(state: HostPlugins, next: Map, failed: readonly string[]): void { + for (const plugin of failed) { + const loaded = state.loaded.get(plugin); + if (loaded === undefined) next.delete(plugin); + else next.set(plugin, loaded); + } + state.loaded.clear(); + for (const [name, each] of next) state.loaded.set(name, each); +} + +/** What the commit of a reload that rejected applied: an `aborted` error after its commit says (`details.committed`). */ +function committedOf(error: unknown): CommittedReload | undefined { + if (!isChordError(error, "aborted")) return undefined; + const details = error.details as { readonly committed?: CommittedReload } | null | undefined; + // The kernel's own error: its `committed` is a `CommittedReload`. + return typeof details === "object" && details !== null ? details.committed : undefined; +} + +/** A plugin's facet passed to a host, or why it was not: a build or evaluation failure. */ +type Prepared = + | { readonly kind: "facet"; readonly facet: Facet; readonly digest: string } + | { readonly kind: "failed"; readonly error: unknown }; + +class PluginsComponent implements Plugins { + readonly #options: PluginsOptions; + readonly #watcher: PluginWatcher; + #discovery: Discovery = { plugins: [], problems: [] }; + #discovered = false; + /** The latest build of every (plugin, kind) built so far, by `plugin\u0000kind`. */ + readonly #built = new Map(); + /** Each loaded host's plugins, held as long as the host is. */ + readonly #hosts = new WeakMap(); + /** The marks each build started from, held as long as its result is (a rollout given a copy clears none). */ + readonly #snapshots = new WeakMap(); + + constructor(options: PluginsOptions) { + this.#options = options; + const batch = new Batch((error) => options.onError(error), undefined); + this.#watcher = new PluginWatcher( + { + roots: options.roots, + base: options.base ?? [], + kinds: options.kinds, + failed: (error) => options.onError({ kind: "watch_failed", error }), + }, + batch, + ); + } + + get state(): Watchable { + // Its value is a `PluginsState`. + return this.#watcher as unknown as Watchable; + } + + get discovery(): Discovery { + return this.#discovery; + } + + async build(names: readonly string[], context: Context): Promise { + const before = new Set(this.#discovery.plugins.map(({ name }) => name)); + if (!this.#discovered || this.#watcher.needsDiscovery) await this.#discover(context); + const now = this.#discovery.plugins; + const removed = [...before].filter((name) => !now.some((plugin) => plugin.name === name)); + const wanted = now.filter(({ name }) => names.includes(name) || !before.has(name)); + const snapshot = this.#watcher.snapshot([...wanted.map(({ name }) => name), ...removed]); + const facets = await Promise.all( + wanted.flatMap((plugin) => Object.keys(plugin.facets).map((kind) => this.#buildOne(plugin, kind, context))), + ); + const plugins = Object.fromEntries(wanted.map((plugin) => [plugin.name, plugin])); + const built: Built = { facets, plugins, removed }; + this.#snapshots.set(built, snapshot); + return built; + } + + async load

( + host: Host

, + options: { readonly kind: string; readonly core: readonly PluginFacet

[] }, + context: Context, + ): Promise { + const { kind } = options; + if (!this.#discovered) await this.#discover(context); + const plugins = this.#discovery.plugins.filter((plugin) => plugin.facets[kind] !== undefined); + const prepared = await Promise.all( + plugins.map(async (plugin) => { + const key = `${plugin.name}\u0000${kind}`; + const built = this.#built.get(key) ?? (await this.#buildOne(plugin, kind, context)); + return [plugin.name, this.#prepare(plugin, built)] as const; + }), + ); + const failed: PluginFailure[] = []; + const facets: PluginFacet

[] = [...options.core]; + for (const [plugin, each] of prepared) { + if (each.kind === "failed") failed.push({ plugin, error: each.error }); + // Evaluated for this host's principal type: the facets are the app's (`Facet

` is invariant in `P`). + else facets.push({ name: plugin, facet: each.facet as unknown as Facet

}); + } + const report = await host.load(facets, context); + const left = new Set(report.failed.map(({ plugin }) => plugin)); + const loaded = new Map(); + for (const [plugin, each] of prepared) { + if (each.kind === "facet" && !left.has(plugin)) loaded.set(plugin, each); + } + // The host's own facets, for its later reloads. + const core = options.core as unknown as readonly PluginFacet[]; + this.#hosts.set(host, { kind, core, loaded, reloading: Promise.resolve() }); + return { failed: [...failed, ...report.failed] }; + } + + reload

(host: Host

, built: Built, context: Context): Promise { + const state = this.#hosts.get(host); + if (state === undefined) throw new Error("plugins.reload: the host was not loaded by this component"); + // One after another per host: a reload computing its set from a loaded map an earlier one is about to change + // would pass that plugin's old facet and roll it back. + const reload = state.reloading.then(() => this.#reload(host, state, built, context)); + // The tail settles to nothing: a report kept as its value would keep a rejected facet's closure (its error). + state.reloading = reload.then( + () => {}, + () => {}, + ); + return reload; + } + + /** + * One reload of `host` with `built`: the plugins it loaded and those `built` brings, without the ones `built` + * removed or rebuilt without a facet of the host's kind (its own discovery's, never a later one's). + */ + async #reload

( + host: Host

, + state: HostPlugins, + built: Built, + context: Context, + ): Promise { + const { kind } = state; + const fresh = new Map(); + for (const facet of built.facets) if (facet.facet === kind) fresh.set(facet.plugin, facet); + const names = [...new Set([...state.loaded.keys(), ...fresh.keys()])].sort(); + const failed: PluginFailure[] = []; + const next = new Map(); + for (const name of names) { + // Own properties only: a plugin may be named like one of `Object.prototype`'s (`constructor`). + const plugin = Object.hasOwn(built.plugins, name) ? built.plugins[name] : undefined; + // Removed by this build's discovery, or rebuilt without a facet of this kind: left out. + if (built.removed.includes(name) || (plugin !== undefined && plugin.facets[kind] === undefined)) continue; + const loaded = state.loaded.get(name); + const script = fresh.get(name); + if ( + script === undefined || + plugin === undefined || + (script.kind === "script" && script.digest === loaded?.digest) + ) { + if (loaded !== undefined) next.set(name, loaded); + continue; + } + const prepared = this.#prepare(plugin, script); + if (prepared.kind === "facet") { + next.set(name, prepared); + continue; + } + failed.push({ plugin: name, error: prepared.error }); + if (loaded !== undefined) next.set(name, loaded); + } + const facets = [...state.core, ...[...next].map(([name, { facet }]) => ({ name, facet }))]; + // The host takes facets of its own principal type: the component loaded them for it. The plugins whose build or + // evaluation failed are the host's to report, and its closure rule's to consider. + let report: ReloadReport; + try { + report = await host.reload(facets as unknown as readonly PluginFacet

[], context, { failed }); + } catch (error) { + // Aborted after its commit: the commit stands, and is what the host runs now. + const committed = committedOf(error); + if (committed !== undefined) adopt(state, next, committed.failed); + throw error; + } + adopt( + state, + next, + report.failed.map(({ plugin }) => plugin), + ); + return report; + } + + step

(name: string, host: Host

): RolloutStep { + const state = this.#hosts.get(host); + if (state === undefined) throw new Error("plugins.step: the host was not loaded by this component"); + return { name, kinds: [state.kind], reload: (built, context) => this.reload(host, built, context) }; + } + + async rollout(steps: readonly RolloutStep[], built: Built, context: Context): Promise { + const done: { readonly name: string; readonly report: ReloadReport }[] = []; + const failed = new Set(); + for (const step of steps) { + let report: ReloadReport; + try { + report = await step.reload(built, context); + } catch (error) { + return { steps: done, stopped: { step: step.name, error } }; + } + for (const { plugin } of report.failed) failed.add(plugin); + done.push({ name: step.name, report }); + } + const snapshot = this.#snapshots.get(built); + if (snapshot !== undefined) { + this.#watcher.applied( + [...snapshot.plugins.keys()].filter((name) => !failed.has(name)), + snapshot, + ); + } + return { steps: done, stopped: undefined }; + } + + async installDependencies(name: string, context: Context): Promise { + if (!this.#discovered) await this.#discover(context); + const plugin = this.#discovery.plugins.find((each) => each.name === name); + if (plugin === undefined) throw new Error(`plugins.installDependencies: no plugin "${name}"`); + await installDependencies(plugin, context, this.#options.install); + this.#watcher.installed(name); + } + + close(): void { + this.#watcher.close(); + } + + /** Discovers the plugins again; the changes of the plugin set and of `package.json`s seen until now clear. */ + async #discover(context: Context): Promise { + const seen = this.#watcher.discoverySnapshot(); + this.#discovery = await discoverPlugins(this.#options.roots, { kinds: this.#options.kinds }, context); + this.#discovered = true; + this.#watcher.know(this.#discovery, seen); + } + + /** Builds `plugin`'s facet of `kind` and keeps it as the latest; the watcher learns what it read. */ + async #buildOne(plugin: Plugin, kind: string, context: Context): Promise { + const { hostPackages, browserKinds = [], publicPath } = this.#options; + const platform = browserKinds.includes(kind) ? "browser" : "node"; + const options = { + hostPackages: Object.keys(hostPackages), + platform, + plugins: this.#discovery.plugins, + ...(publicPath === undefined ? {} : { publicPath }), + } as const; + const built = await buildFacet(plugin, kind, options, context); + this.#built.set(`${plugin.name}\u0000${kind}`, built); + this.#watcher.used([built]); + return built; + } + + /** `built` evaluated for `plugin`, or why not: its build failed, or its evaluation threw. */ + #prepare(plugin: Plugin, built: BuiltFacet): Prepared { + if (built.kind === "failed") { + const message = `plugin "${plugin.name}": its ${built.facet} facet failed to build:\n${built.errors.join("\n")}`; + return { kind: "failed", error: new Error(message) }; + } + try { + const facet = evaluateFacet(built, { hostPackages: this.#options.hostPackages, plugin }); + return { kind: "facet", facet, digest: built.digest }; + } catch (error) { + return { kind: "failed", error }; + } + } +} diff --git a/packages/chord/src/next-node/plugins/stack.ts b/packages/chord/src/next-node/plugins/stack.ts new file mode 100644 index 000000000..c82c47c17 --- /dev/null +++ b/packages/chord/src/next-node/plugins/stack.ts @@ -0,0 +1,67 @@ +// Stack traces of plugin scripts (slice 7c §3, decision Q1 (b)): `vm.Script` maps no positions through a source map, so +// every evaluated script registers its map under its filename while something of its evaluation lives (its facet, or +// the error it threw while evaluated), and `mapStack` rewrites an error's stack through them, for the app's reporters. +// Nothing global: no `Error.prepareStackTrace`, no process-wide source-map switch. The registry is a module-level `Map` +// keyed by filename, not a `WeakMap` keyed by an owner: `mapStack` starts from a stack's text, which names only the +// filename. Its entries go with the last evaluation of their script (a `FinalizationRegistry` counts them down). + +import { SourceMap } from "node:module"; +import { basename, join } from "node:path"; + +/** A registered script: how its positions map, its plugin's directory, and how many of its evaluations live. */ +type Registered = { readonly map: SourceMap; readonly dir: string; live: number }; + +/** The evaluated scripts by filename, while an evaluation of each lives. */ +const scripts = new Map(); + +/** Counts down an evaluation that was collected; its script's entry goes with the last one. */ +const collected = new FinalizationRegistry((filename) => { + const script = scripts.get(filename); + if (script !== undefined && --script.live === 0) scripts.delete(filename); +}); + +/** + * Registers the script `source` evaluated under `filename` for plugin directory `dir` while `owner` (its facet, or the + * error its evaluation threw) lives: its inline source map, whose sources are relative to the plugin's directory. Two + * evaluations of one script (two hosts) share the entry. + */ +export function registerScript(filename: string, source: string, dir: string, owner: object): void { + let script = scripts.get(filename); + if (script === undefined) { + const map = inlineSourceMap(source); + if (map === undefined) return; + script = { map, dir, live: 0 }; + scripts.set(filename, script); + } + script.live++; + collected.register(owner, filename); +} + +/** The inline source map esbuild appended to a built script, if it has one. */ +export function inlineSourceMap(source: string): SourceMap | undefined { + const inline = SOURCE_MAP.exec(source); + return inline === null ? undefined : new SourceMap(JSON.parse(Buffer.from(inline[1]!, "base64").toString("utf8"))); +} + +/** The inline source map esbuild appends to a script. */ +const SOURCE_MAP = /\/\/# sourceMappingURL=data:application\/json;base64,([A-Za-z0-9+/=]+)\s*$/; + +/** + * `error`'s stack with every position in a registered plugin script mapped through its source map to the plugin's file + * and line (`notes/worker.ts:12:3`: the plugin's directory name and the file's path inside it); other lines as they + * are. A value that is no error with a stack gives its string. + */ +export function mapStack(error: unknown): string { + const stack = error instanceof Error ? error.stack : undefined; + if (stack === undefined) return String(error); + return stack.replace(POSITION, (whole, filename: string, line: string, column: string) => { + const script = scripts.get(filename); + const entry = script?.map.findEntry(Number(line) - 1, Number(column) - 1); + if (script === undefined || entry === undefined || !("originalSource" in entry)) return whole; + const file = join(basename(script.dir), entry.originalSource).split("\\").join("/"); + return `${file}:${entry.originalLine + 1}:${entry.originalColumn + 1}`; + }); +} + +/** A position in a stack line: `filename:line:column`, the filename being anything up to it (spaces included). */ +const POSITION = /([^\s()]+):(\d+):(\d+)/g; diff --git a/packages/chord/src/next-node/plugins/watcher.ts b/packages/chord/src/next-node/plugins/watcher.ts new file mode 100644 index 000000000..b4bb6f19b --- /dev/null +++ b/packages/chord/src/next-node/plugins/watcher.ts @@ -0,0 +1,300 @@ +// The watcher and its marks (slice 7c §5): `fs.watch` on every directory of the plugin roots and the app's base code, +// `node_modules` and `.git` never watched (a directory appearing later gets its watch), marks what changed and never +// applies it. A changed file dirties every plugin whose facets used it (a copied contract every user) or whose directory +// contains it; a changed `package.json` asks for an install (and for its metadata to be read again); a plugin directory +// or facet file added or removed asks for discovery again; a file of base code asks the app to restart its hosts. The +// marks are a JSON watchable; each mark records the generation of its latest change (generations only increase), so a +// change during a build keeps its mark, however late an older build's rollout clears. + +import { type Dirent, type FSWatcher, lstatSync, readdirSync, realpathSync, watch } from "node:fs"; +import { basename, dirname, join, resolve, sep } from "node:path"; +import { BACKGROUND_CONTEXT, type Context } from "../../context/index.ts"; +import type { JsonValue } from "../../json.ts"; +import { type Batch, Source } from "../../next/state/internal.ts"; +import { type BuiltFacet, inside } from "./build.ts"; +import { type Discovery, FACET_EXTENSIONS } from "./discover.ts"; + +/** The marks: plugins to rebuild, plugins to install, whether to discover again, whether base code changed. */ +export type PluginsState = { + readonly dirty: readonly string[]; + readonly install: readonly string[]; + readonly set: boolean; + readonly base: boolean; +}; + +/** The generation of each dirty plugin's latest change when a build started: what its rollout may clear. */ +export type MarkSnapshot = { readonly plugins: ReadonlyMap }; + +/** The generations of the latest change of the plugin set and of a `package.json` when a discovery started. */ +export type DiscoverySnapshot = { readonly set: number; readonly manifest: number }; + +/** What the watcher watches and how it reports: the roots, the base code, the facet kinds, a failure to watch. */ +export type WatcherOptions = { + readonly roots: readonly string[]; + readonly base: readonly string[]; + readonly kinds: readonly string[]; + /** The watcher cannot watch (the system's limit): reported once, after which the app builds on demand. */ + readonly failed: (error: unknown) => void; +}; + +/** Directories never watched, and changes under them never marked. */ +const IGNORED = new Set(["node_modules", ".git"]); + +/** + * The watcher and the marks (`state`), a JSON watchable published at most once per microtask turn with changes. It + * marks for the plugins of the last discovery (`know`) and the files their last builds read (`used`). + */ +export class PluginWatcher extends Source { + readonly #options: WatcherOptions; + readonly #roots: readonly string[]; + readonly #baseDirs: readonly string[]; + /** The trees it watches: the roots, the base code, and plugin directories linked from outside the roots. */ + readonly #trees = new Set(); + /** Its watches, one per directory of its trees. */ + readonly #watchers = new Map(); + /** A failure to watch was reported (once). */ + #reported = false; + /** The generation the next change gets: every mark records a newer one than any before it. */ + #generation = 1; + /** Each dirty plugin's latest change, by generation. */ + readonly #dirty = new Map(); + readonly #install = new Set(); + /** The latest change of the plugin set since the last discovery (0: none). */ + #set = 0; + /** The latest change of a known plugin's `package.json` since the last discovery (0: none). */ + #manifest = 0; + #baseChanged = false; + #discovery: Discovery = { plugins: [], problems: [] }; + /** The files each built facet read (its inputs and copies), by plugin and kind. */ + readonly #used = new Map }>(); + #value: JsonValue | undefined; + #scheduled = false; + + constructor(options: WatcherOptions, batch: Batch) { + super(batch); + this.#options = options; + // Events name paths under the directories as watched: by their real paths, as discovery names plugins. + this.#roots = options.roots.map(real); + this.#baseDirs = options.base.map(real); + for (const dir of [...this.#roots, ...this.#baseDirs]) this.#watchTree(dir); + } + + get value(): JsonValue { + this.#value ??= { + dirty: [...this.#dirty.keys()].sort(), + install: [...this.#install].sort(), + set: this.#set > 0, + base: this.#baseChanged, + }; + return this.#value; + } + + deliver(context: Context): void { + const value = this.value; + this.publish(value, [["r", value]], false, context); + } + + /** Whether the next build discovers again: the plugin set or a plugin's `package.json` changed. */ + get needsDiscovery(): boolean { + return this.#set > 0 || this.#manifest > 0; + } + + /** What a discovery that starts now has seen. */ + discoverySnapshot(): DiscoverySnapshot { + return { set: this.#set, manifest: this.#manifest }; + } + + /** The plugins discovery found: marks are for them (a removed one's install clears); the changes it had seen clear. */ + know(discovery: Discovery, seen: DiscoverySnapshot): void { + this.#discovery = discovery; + if (this.#set === seen.set) this.#set = 0; + if (this.#manifest === seen.manifest) this.#manifest = 0; + // A removed plugin has nothing to install (its directory went, its package.json with it). + for (const name of this.#install) + if (!discovery.plugins.some((plugin) => plugin.name === name)) this.#install.delete(name); + // A plugin directory linked from outside the roots: a tree of its own (a symlink is not followed). + const linked = discovery.plugins.filter(({ dir }) => !this.#roots.some((root) => inside(root, dir))); + for (const { dir } of linked) this.#watchTree(dir); + for (const tree of this.#trees) { + const kept = + this.#roots.includes(tree) || this.#baseDirs.includes(tree) || linked.some((each) => each.dir === tree); + if (!kept) this.#unwatchTree(tree); + } + this.#changed(); + } + + /** What `built` facets read: a later change of one of these files dirties their plugins. */ + used(built: readonly BuiltFacet[]): void { + for (const facet of built) { + if (facet.kind !== "script") continue; + const files = new Set([...facet.inputs.map(({ file }) => file), ...facet.copied]); + this.#used.set(`${facet.plugin}\u0000${facet.facet}`, { plugin: facet.plugin, files }); + } + } + + /** The marks a build of `names` starts from: each one's latest change. */ + snapshot(names: Iterable): MarkSnapshot { + const plugins = new Map(); + for (const name of names) plugins.set(name, this.#dirty.get(name) ?? 0); + return { plugins }; + } + + /** `names` were applied everywhere: their marks clear unless a change came after `snapshot`. */ + applied(names: Iterable, snapshot: MarkSnapshot): void { + for (const name of names) { + const latest = this.#dirty.get(name); + if (latest !== undefined && latest <= (snapshot.plugins.get(name) ?? 0)) this.#dirty.delete(name); + } + this.#changed(); + } + + /** Plugin `name`'s dependencies were installed: its install mark clears and it is dirty (its packages changed). */ + installed(name: string): void { + this.#install.delete(name); + this.#dirty.set(name, this.#generation++); + this.#changed(); + } + + /** Marks what a change of the file or directory at `path` (absolute) asks for. */ + mark(path: string): void { + if (path.split(sep).some((part) => IGNORED.has(part))) return; + if (this.#baseDirs.some((dir) => inside(dir, path))) { + this.#baseChanged = true; + } else { + this.#markPlugin(path, this.#generation++); + } + this.#changed(); + } + + /** Stops watching. */ + close(): void { + for (const watcher of this.#watchers.values()) watcher.close(); + this.#watchers.clear(); + this.#trees.clear(); + } + + /** + * A change under a root, at `generation`: in a plugin's directory, it dirties that plugin (and asks for an install + * and a new reading of its metadata for its `package.json`, for discovery for a facet file at its top); elsewhere + * under a root (a directory added or removed, a directory that is no plugin yet), discovery; every plugin whose + * facets read the file is dirty too. + */ + #markPlugin(path: string, generation: number): void { + const plugin = this.#discovery.plugins.find((each) => each.dir === path || inside(each.dir, path)); + if (plugin === undefined) { + if (this.#roots.some((root) => inside(root, path))) this.#set = generation; + } else if (path === join(plugin.dir, "package.json")) { + this.#install.add(plugin.name); + this.#manifest = generation; + } else { + if (path === plugin.dir || (dirname(path) === plugin.dir && this.#isFacetFile(basename(path)))) { + this.#set = generation; + } + this.#dirty.set(plugin.name, generation); + } + for (const { plugin: user, files } of this.#used.values()) if (files.has(path)) this.#dirty.set(user, generation); + } + + /** Whether `file` (at a plugin's top) is named as a facet file of one of the app's kinds. */ + #isFacetFile(file: string): boolean { + return this.#options.kinds.some((kind) => FACET_EXTENSIONS.some((extension) => file === `${kind}${extension}`)); + } + + /** Watches the tree at `dir` (once): it and every directory under it but ignored ones; a missing one is skipped. */ + #watchTree(dir: string): void { + if (this.#trees.has(dir)) return; + this.#trees.add(dir); + this.#watchDirs(dir); + } + + /** Stops watching the tree at `dir`. */ + #unwatchTree(dir: string): void { + this.#trees.delete(dir); + this.#unwatchDirs(dir); + } + + /** Watches `dir` and, walking down, every directory under it but ignored ones (each not watched yet). */ + #watchDirs(dir: string): void { + if (this.#watchers.has(dir) || this.#reported) return; + let watcher: FSWatcher; + try { + watcher = watch(dir, (_event, file) => { + if (file !== null) this.#changedIn(join(dir, file)); + }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") this.#failed(error); + return; + } + watcher.on("error", (error) => this.#failed(error)); + this.#watchers.set(dir, watcher); + let entries: Dirent[]; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + // Removed meanwhile: its parent's event says so. + return; + } + for (const entry of entries) { + if (entry.isDirectory() && !IGNORED.has(entry.name)) this.#watchDirs(join(dir, entry.name)); + } + } + + /** An event for `path` in a watched directory: a new directory gets its watches, a removed one loses them; marked. */ + #changedIn(path: string): void { + const name = basename(path); + if (!IGNORED.has(name)) { + if (isDirectory(path)) this.#watchDirs(path); + else if (this.#watchers.has(path)) this.#unwatchDirs(path); + } + this.mark(path); + } + + /** Stops watching `dir` and the directories under it (it was removed, or its tree is not watched any more). */ + #unwatchDirs(dir: string): void { + for (const [watched, watcher] of this.#watchers) { + if (watched !== dir && !inside(dir, watched)) continue; + watcher.close(); + this.#watchers.delete(watched); + } + } + + /** Watching failed: reported once; every watch stops (the app builds on demand). */ + #failed(error: unknown): void { + this.close(); + if (this.#reported) return; + this.#reported = true; + this.#options.failed(error); + } + + /** The marks changed: one publication per microtask turn while someone watches or derives from them. */ + #changed(): void { + this.#value = undefined; + if (this.#scheduled || (!this.watched && this.dependents.size === 0)) return; + this.#scheduled = true; + queueMicrotask(() => { + this.#scheduled = false; + this.batch.within(() => this.changed(), BACKGROUND_CONTEXT); + }); + } +} + +/** + * Whether `path` is a directory, never following a link (as the scan of a tree does not); one that cannot be read (gone, + * a link loop, no access) is none: a watch callback must not throw. + */ +function isDirectory(path: string): boolean { + try { + return lstatSync(path, { throwIfNoEntry: false })?.isDirectory() === true; + } catch { + return false; + } +} + +/** `dir` absolute and by its real path; as given (absolute) when it does not exist. */ +function real(dir: string): string { + try { + return realpathSync(dir); + } catch { + return resolve(dir); + } +} diff --git a/packages/chord/src/next/host/README.md b/packages/chord/src/next/host/README.md index 9844b33b0..54586d282 100644 --- a/packages/chord/src/next/host/README.md +++ b/packages/chord/src/next/host/README.md @@ -247,7 +247,21 @@ server.accept(webSocketLink(ws), { principal, limits: { part: 16 * MiB, decoded: resolves when each has its first outcome (decision (a)): live, failed by its own cause, parked with no blocker that is an object here still without its first outcome (a keyed pin still loading counts as here; a wait on another host or on a failed dependency ends the first pass), or ended (`close` settles a waiting `load`); later retries never - hold it. A setup that throws rejects it with that error and closes the host as `close` does, its links included. + hold it. Per plugin (decision "Per plugin at startup and reload"): a plugin whose setup throws or is refused at + declaration (an invalid or repeated name, a type or registry an earlier plugin serves, a remote type here without + principal or address, `checkAccepts`) is left out; `load` resolves with `{ failed: [{ plugin, error }] }` and never + closes the host; its consumers wait `failed { waiting }`. +- `host.reload([{ name, facet }], ctx)` (slice 7b, per plugin since 7c): the complete new plugin set; staging runs the + new code per plugin (unchanged and failed plugins keep their loaded entries, their setups not run again). A plugin + whose new code fails (its setup, a factory against the recording `self`, a declaration error, a name another plugin + serves) keeps its old code (an added one is left out) and staging runs again without it; then a changed plugin whose + recorded requests would not bind because of such a plugin (its old code serving the name, or nobody serving it) + keeps its old code too (the closure rule), to a fixpoint; a request is checked as a bind checks it, the provider's + host address included. `reload(facets, ctx, { failed })` names plugins the caller passes as their loaded facet + because their new code failed before (a build, an evaluation: the plugins component): the same for the closure rule. + The report lists them all in `failed`; the rest commits. `ctx`'s abort before the commit rejects `aborted` and + changes nothing; after it, it ends only the wait for the report: `aborted` with `details.committed` + (`CommittedReload`: `added`, `changed`, `removed`, `failed` by plugin name), the commit standing. - `host.hold(Type, params, ctx, { readyTimeout? })`: a reference at once (`Held`: its `variant` is `undefined` until its first binding), which follows its object until released and never rejects: `loading` before the publication, while the object activates, retries or is parked on dependencies under way, while there is no parent link or it is diff --git a/packages/chord/src/next/host/destination.ts b/packages/chord/src/next/host/destination.ts index 8a372b257..9f8633d7f 100644 --- a/packages/chord/src/next/host/destination.ts +++ b/packages/chord/src/next/host/destination.ts @@ -43,6 +43,14 @@ export class Destination { return { kind: "path", path: this.#pathTo(address, principal, context) ?? this.#parent, address }; } + /** + * Whether a reference to `Type` with `params` goes here: a local type, or a remote one whose host address is this + * host's. A `hostAddress` that throws, throws. + */ + placedHere(Type: Contract, params: ParamsOf): boolean { + return Type.kind === "local" || this.#isHere(hostAddressOf(Type, params)); + } + /** * Where a hold arriving on `link` acting as `principal` goes, by the address it names: here when it is this host's * (a relay never captures a hold for serving the same name); over the route of its kind; else over the parent link diff --git a/packages/chord/src/next/host/host.ts b/packages/chord/src/next/host/host.ts index a8737b7d4..50d6a94e6 100644 --- a/packages/chord/src/next/host/host.ts +++ b/packages/chord/src/next/host/host.ts @@ -56,7 +56,19 @@ import { Binding, type CheckedParams, type Held, type Reference, type Target } f import { RevocationKeys, type RevokedBy } from "./revocation.ts"; import { type RootOptions, RootScopes } from "./root.ts"; import { Route, type RouteOptions } from "./route.ts"; -import { type PluginFacet, recordFactory, type Staged, type StagingHost, stagePlugins } from "./staging.ts"; +import { + addStaged, + emptyStaged, + type PluginFacet, + type Recorded, + type Request, + recordFactory, + type Staged, + type StagedPlugin, + type StagedRegistry, + type StagingHost, + stagePlugin, +} from "./staging.ts"; import { UnboundIndex, type UnboundRequest } from "./unbound.ts"; import { uuidv7 } from "./uuid.ts"; @@ -201,22 +213,28 @@ export interface Host

[], context: Context): Promise; + load(facets: readonly PluginFacet

[], context: Context): Promise; /** * Reloads this host's plugins (slice 7b): `facets` is its complete new plugin set; a plugin in it and not loaded * is added, one loaded with another facet value is changed (the same facet object is unchanged: untouched), one - * loaded and not in it is removed. Staging runs every setup and every affected object's new factory against a - * recording `self`; any throw rejects it and nothing changed. Then it commits in one batch (types, registries, + * loaded and not in it is removed. Staging runs every new setup and every affected object's new factory against a + * recording `self`, per plugin: a plugin whose new code fails there keeps its old code (an added one is left out), + * and so does a changed plugin whose recorded requests would not bind because of it (the closure rule); both are + * reported (`failed`); `options.failed` names plugins the caller passes as their loaded facet because their new + * code failed before (a build, an evaluation): the same for the closure rule and the report. Then it commits in one + * batch (types, registries, * contributions; a publication waking the requests for the names it serves) and every object of a changed or * removed plugin takes its fate: swapped in place, restarted after its drain, retried, or ended (its type * removed). Resolves with the report once every fate started (swaps and retries checked their dependencies; it * does not wait for objects to go live). Reloads run one after another. Throws before `load` resolved; rejects * `released/closed` after `close`; `context`'s abort before the commit rejects `aborted` and changes nothing, after - * it only ends the wait. + * it only ends the wait (`aborted`, its `details.committed` naming what the commit applied: `CommittedReload`). */ - reload(facets: readonly PluginFacet

[], context: Context): Promise; + reload(facets: readonly PluginFacet

[], context: Context, options?: ReloadOptions): Promise; /** * A reference once its object is live (and hydrated). It acts as the host's principal and goes where its type's host * address puts it: here, over a route, or over the parent link (`Destination`). It waits while the reference is @@ -294,13 +312,38 @@ export interface Host

; } -/** What a reload did (slice 7b §1): its plugins by change, and what every affected object did. */ +/** A plugin left out at `load`, or whose new code failed at a reload (it kept its old code, if it had any), and why. */ +export type PluginFailure = { readonly plugin: string; readonly error: unknown }; + +/** + * What a reload's commit applied, by plugin name: the `details.committed` of the `aborted` error of a reload whose + * context aborted after its commit (the commit stands; only the wait for the report ended). + */ +export type CommittedReload = { + readonly added: readonly string[]; + readonly changed: readonly string[]; + readonly removed: readonly string[]; + /** The plugins whose new code failed: they kept their old code (an added one was left out). */ + readonly failed: readonly string[]; +}; + +/** What `host.reload` takes besides the plugin set: the plugins whose new code failed before it (a build, an evaluation). */ +export type ReloadOptions = { readonly failed?: readonly PluginFailure[] }; + +/** What `load` did (decision "Per plugin at startup and reload"): the plugins it left out. */ +export type LoadReport = { readonly failed: readonly PluginFailure[] }; + +/** + * What a reload did (slice 7b §1): its plugins by change (those applied), the plugins whose new code failed, and what + * every affected object did. + */ export type ReloadReport = { readonly plugins: { readonly added: readonly string[]; readonly changed: readonly string[]; readonly removed: readonly string[]; }; + readonly failed: readonly PluginFailure[]; /** Every live, activating, parked or failed object of a changed or removed plugin, by its label. */ readonly objects: readonly ReloadedObject[]; }; @@ -326,23 +369,38 @@ export type Serving = | { readonly kind: "object"; readonly object: ChordObject } | { readonly kind: "waiting"; readonly cause: FailureCause }; -/** A published plugin: its facet (an unchanged reload passes the same object), its registries and contributions. */ +/** + * A published plugin: its facet (an unchanged reload passes the same object), what its setup staged (a reload that + * keeps it uses it again, its setup not run) and its contributions in the registries' tables. + */ type LoadedPlugin

= { readonly facet: Facet

; - readonly registries: readonly string[]; + readonly staged: StagedPlugin; readonly contributions: readonly RegistryContribution[]; }; -/** What a reload staged: the staged set, the plugins by change, and each affected object's fate. */ +/** + * What a reload staged: the staged set, the facets of the plugins it applies, the plugins by change, those that failed, + * and each affected object's fate. + */ type ReloadPlan

= { readonly staged: Staged; readonly facets: ReadonlyMap>; readonly added: readonly string[]; readonly changed: readonly string[]; readonly removed: readonly string[]; + readonly failed: readonly PluginFailure[]; readonly fates: readonly PlannedFate[]; }; +/** One staging pass of a reload: its plan, or the plugins it found failing (staging runs again without their new code). */ +type StagingPass

= + | { readonly kind: "staged"; readonly plan: ReloadPlan

; readonly recorded: readonly RecordedFactory[] } + | { readonly kind: "failed"; readonly failed: ReadonlyMap }; + +/** A recorded new factory of an object staging planned a fate for: whose new code it is, and what it declared. */ +type RecordedFactory = { readonly plugin: string; readonly recorded: Recorded }; + /** An affected object's fate, as staging decided it. */ type PlannedFate = | { readonly kind: "end"; readonly object: ChordObject } @@ -389,7 +447,7 @@ export class ChordHost

>(); /** The reloads, one after another (slice 7b §1): the last one's end, which the next one waits for. */ - #reloading: Promise = Promise.resolve(); + #reloading: Promise = Promise.resolve(); /** By identity, in the order they went live (the others where they were created). */ readonly #objects = new Map(); /** Teardowns still running, by identity: the next instance of an identity waits for its previous ones. */ @@ -443,26 +501,31 @@ export class ChordHost

this.#storageFailed(error), batch: this.batch }); } - async load(facets: readonly PluginFacet

[], context: Context): Promise { + async load(facets: readonly PluginFacet

[], context: Context): Promise { this.batch.assertOutside("load"); this.#assertOpen(); if (this.#state !== "new") throw new Error("load runs once per host"); this.#state = "loading"; - try { - const staged = stagePlugins(facets, this.#stagingHost()); - this.#types = staged.types; - // The publication: the types and registries served, the facets' contributions and the first composition of - // every served registry, in one batch, before any object activates. - this.batch.commit(() => { - this.#publish(staged, staged.contributions.keys(), facetsByName(facets)); - for (const record of this.#registries.values()) record.composeServed(); - }, context); - } catch (error) { - // The host closes as `close` would: its links too. Nothing was activated, so no close can fail. - this.#closing ??= this.#close(); - await this.#closing.catch(() => {}); - throw error; + const staged = emptyStaged(); + const loaded = new Map>(); + const failed: PluginFailure[] = []; + // A name given twice is refused the second time whether or not the first staged. + const given = new Set(); + // In the given order: of two plugins serving one name, the later is left out. + for (const plugin of facets) { + const twice = given.has(plugin.name); + given.add(plugin.name); + const error = twice ? loadedTwice(plugin.name) : this.#stageInto(staged, plugin); + if (error === undefined) loaded.set(plugin.name, plugin.facet); + else failed.push({ plugin: plugin.name, error }); } + this.#types = staged.types; + // The publication: the types and registries served, the facets' contributions and the first composition of + // every served registry, in one batch, before any object activates. + this.batch.commit(() => { + this.#publish(staged, loaded); + for (const record of this.#registries.values()) record.composeServed(); + }, context); // After the batch: what waited for the publication is placed (the gate lasted only for the setups). this.#published = true; this.#publication.resolve(); @@ -472,48 +535,133 @@ export class ChordHost

this.#object(served, EMPTY_PARAMS, undefined, context)); await Promise.all(objects.map(({ firstOutcome }) => firstOutcome)); if (this.#state === "loading") this.#state = "loaded"; + return { failed }; } - reload(facets: readonly PluginFacet

[], context: Context): Promise { + /** + * Whether a request for `Type` with `params` binds to `served` as a real bind checks it (`bind`): its kind, version + * and fingerprint, and the provider's own host address; one whose `hostAddress` throws does not. + */ + #wouldBind(served: Served, Type: Contract, params: ParamsOf): boolean { + if (unboundCause(served, Type.kind, Type.name, Type.version, Type.fingerprint) !== undefined) return false; + try { + return this.#misplaced(Type, served, params) === undefined; + } catch { + return false; + } + } + + /** Whether a request for `Type` with `params` goes here; a `hostAddress` that throws: not here (it fails at bind). */ + #placedHere(Type: Contract, params: ParamsOf): boolean { + try { + return this.#destination.placedHere(Type, params); + } catch { + return false; + } + } + + /** Stages `plugin`'s new code into `staged`: the error that refuses it (its setup, a declaration, a name served twice). */ + #stageInto(staged: Staged, plugin: PluginFacet

): unknown { + try { + return addStaged(staged, stagePlugin(plugin, this.#stagingHost())); + } catch (error) { + return error; + } + } + + reload(facets: readonly PluginFacet

[], context: Context, options: ReloadOptions = {}): Promise { this.batch.assertOutside("reload"); if (this.#state === "closed") return Promise.reject(this.#closedError()); if (this.#state !== "loaded") throw new Error("reload runs after load"); - const reload = this.#reloading.then(() => this.#reload(facets, context)); - this.#reloading = reload.catch(() => {}); + const reload = this.#reloading.then(() => this.#reload(facets, options.failed ?? [], context)); + // The tail settles to nothing: a report kept as its value would keep a rejected facet's closure (its error). + this.#reloading = reload.then( + () => {}, + () => {}, + ); return reload; } /** One reload, after the previous one: staging, the commit, the fates, the report. */ - async #reload(facets: readonly PluginFacet

[], context: Context): Promise { + async #reload( + facets: readonly PluginFacet

[], + before: readonly PluginFailure[], + context: Context, + ): Promise { this.#assertOpen(); if (context.abortSignal?.aborted) throw reloadAborted(); - const plan = this.#stageReload(facets, context); - // Aborted while it staged (a setup may abort it synchronously): rejected as a staging error, nothing changed. + const plan = this.#stageReload(facets, before, context); + // Aborted while it staged (a setup may abort it synchronously): nothing changed. if (context.abortSignal?.aborted) throw reloadAborted(); const started = this.#commit(plan, context); - const objects = await this.#report(started, context); - const { added, changed, removed } = plan; - return { plugins: { added, changed, removed }, objects }; + const { added, changed, removed, failed } = plan; + const plugins = { added, changed, removed }; + const committed: CommittedReload = { ...plugins, failed: failed.map(({ plugin }) => plugin) }; + const objects = await this.#report(started, committed, context); + return { plugins, failed, objects }; } /** - * Staging (slice 7b §2), synchronous (no request or arriving hold meets it half done): every setup runs into a - * staged table (`stagePlugins`), every object of a changed or removed plugin whose type is still served has its new - * factory recorded, which decides its fate. A throw rejects the reload with nothing changed. + * Staging (slice 7b §2), synchronous (no request or arriving hold meets it half done), per plugin (decision "Per + * plugin at startup and reload"): a pass stages the new code (`#stagePass`); a plugin whose new code fails there + * (or failed `before`, passed as its loaded facet) keeps its old code (an added one is left out) and staging runs + * again without it; then a changed plugin whose recorded requests would not bind because of such a plugin keeps its + * old code too (`#closure`), until nothing more fails. Bounded by the number of plugins with new code. */ - #stageReload(facets: readonly PluginFacet

[], context: Context): ReloadPlan

{ - const staged = stagePlugins(facets, this.#stagingHost()); - const byName = facetsByName(facets); - const added: string[] = []; - const changed: string[] = []; - const removed = [...this.#plugins.keys()].filter((name) => !byName.has(name)); - for (const [name, facet] of byName) { - const loaded = this.#plugins.get(name); - if (loaded === undefined) added.push(name); - else if (loaded.facet !== facet) changed.push(name); + #stageReload(facets: readonly PluginFacet

[], before: readonly PluginFailure[], context: Context): ReloadPlan

{ + const failed = new Map(before.map(({ plugin, error }) => [plugin, error])); + const given = new Map>(); + // A name given twice: the second is refused, the first staged. + const twice: PluginFailure[] = []; + for (const { name, facet } of facets) { + if (given.has(name)) twice.push({ plugin: name, error: loadedTwice(name) }); + else given.set(name, facet); } - const reloaded = new Set([...changed, ...removed]); + for (;;) { + const pass = this.#stagePass(given, failed, context); + if (pass.kind === "failed") { + for (const [name, error] of pass.failed) failed.set(name, error); + continue; + } + const kept = this.#closure(pass.plan, pass.recorded, failed); + if (kept.size > 0) { + for (const [name, error] of kept) failed.set(name, error); + continue; + } + const failures = [...failed].map(([plugin, error]) => ({ plugin, error })); + return { ...pass.plan, failed: [...twice, ...failures] }; + } + } + + /** + * One staging pass: the loaded entries of the plugins kept as they are (unchanged, or failed: their setups not run + * again) first, then every other plugin's new code in the given order (one refused by a name the table has already + * fails), then the new factory of every object of a changed or removed plugin whose type is still served (a throw + * fails the plugin serving it). Its plan, or the plugins that failed. + */ + #stagePass( + given: ReadonlyMap>, + failed: ReadonlyMap, + context: Context, + ): StagingPass

{ + const staged = emptyStaged(); + const failing = new Map(); + const fresh: PluginFacet

[] = []; + for (const [name, facet] of given) { + const loaded = this.#plugins.get(name); + if (loaded !== undefined && (loaded.facet === facet || failed.has(name))) addStaged(staged, loaded.staged); + else if (!failed.has(name)) fresh.push({ name, facet }); + } + for (const plugin of fresh) { + const error = this.#stageInto(staged, plugin); + if (error !== undefined) failing.set(plugin.name, error); + } + if (failing.size > 0) return { kind: "failed", failed: failing }; + const applied = new Set(fresh.map(({ name }) => name)); + const removed = [...this.#plugins.keys()].filter((name) => !given.has(name)); + const reloaded = new Set([...applied, ...removed]); const fates: PlannedFate[] = []; + const recorded: RecordedFactory[] = []; for (const object of this.#objects.values()) { if (!reloaded.has(object.served.plugin)) continue; const served = staged.types.get(object.served.contract.name); @@ -521,9 +669,70 @@ export class ChordHost

0) return { kind: "failed", failed: failing }; + const facets = new Map(fresh.map(({ name, facet }) => [name, facet])); + const added = fresh.filter(({ name }) => !this.#plugins.has(name)).map(({ name }) => name); + const changed = fresh.filter(({ name }) => this.#plugins.has(name)).map(({ name }) => name); + const plan = { staged, facets, added, changed, removed, failed: [], fates }; + return { kind: "staged", plan, recorded }; + } + + /** + * The closure rule: the changed plugins (new code applied) of `plan` with a recorded request that would not bind in + * the staged table because of a plugin that failed in this reload: its name served there by such a plugin's old code, + * or not served at all while some plugin failed (what a failed setup would have served is unknown). Requests to + * another host are not checked (a rollout's rules). Each such plugin with the error it keeps its old code for. + */ + #closure( + plan: ReloadPlan

, + recorded: readonly RecordedFactory[], + failed: ReadonlyMap, + ): Map { + const kept = new Map(); + if (failed.size === 0) return kept; + for (const { plugin, recorded: declared } of recorded) { + if (kept.has(plugin) || !plan.facets.has(plugin)) continue; + for (const request of declared.requests) { + const unbound = this.#unboundRequest(plan.staged, request, failed); + if (unbound === undefined) continue; + kept.set(plugin, new Error(`plugin "${plugin}": ${unbound}`)); + break; + } + } + return kept; + } + + /** + * Why `request` (of a new factory) would not bind in `staged` because of a plugin in `failed`; `undefined` + * when it binds, is placed on another host, or does not bind for another reason (it waits as any request does). + */ + #unboundRequest(staged: Staged, request: Request, failed: ReadonlyMap): string | undefined { + const name = request.kind === "type" ? request.contract.name : request.registry.name; + const provider = request.kind === "type" ? staged.types.get(name)?.plugin : staged.registries.get(name)?.plugin; + if (provider !== undefined && !failed.has(provider)) return undefined; + if (request.kind === "type") { + const { contract, params } = request; + // A remote type the object reaches on another host: placed by the rollout's rules, not this host's. + if (!this.#placedHere(contract, params)) return undefined; + const served = staged.types.get(name); + if (served !== undefined && this.#wouldBind(served, contract, params)) return undefined; + return provider === undefined + ? `needs ${name} v${contract.version}, which no plugin serves now (a plugin failed)` + : `needs ${name} v${contract.version}, which plugin "${provider}" serves on its old code`; + } + const served = staged.registries.get(name); + if (served !== undefined && registryServes(served, request.registry.version)) return undefined; + return provider === undefined + ? `needs registry ${name} v${request.registry.version}, which no plugin serves now (a plugin failed)` + : `needs registry ${name} v${request.registry.version}, which plugin "${provider}" serves on its old code`; } /** @@ -534,26 +743,25 @@ export class ChordHost

, context: Context): readonly StartedFate[] { - const { staged, added, changed, removed } = plan; - const published = new Set([...changed, ...added]); - const types = new Map(); - // An unchanged plugin's types stay as they are: its objects are untouched. - for (const [name, served] of staged.types) { - types.set(name, published.has(served.plugin) ? served : this.#types.get(name)!); - } + const { staged, changed, removed } = plan; + const published = new Set(plan.facets.keys()); + // A kept plugin's types are its loaded ones, as they are: its objects are untouched. + const types = staged.types; this.batch.commit(() => { for (const name of [...changed, ...removed]) { const loaded = this.#plugins.get(name)!; - for (const registry of loaded.registries) { + for (const registry of loaded.staged.registries.keys()) { if (staged.registries.get(registry)?.plugin !== name) this.registryRecord(registry).unserve(); } for (const contribution of loaded.contributions) contribution.record.remove(contribution); this.#plugins.delete(name); } this.#types = types; - this.#publish(staged, published, plan.facets); + this.#publish(staged, plan.facets); for (const name of published) { - for (const registry of this.#plugins.get(name)!.registries) this.registryRecord(registry).composeServed(); + for (const registry of staged.plugins.get(name)!.registries.keys()) { + this.registryRecord(registry).composeServed(); + } } }, context); for (const name of this.#references.keys()) if (!types.has(name)) this.#references.delete(name); @@ -588,9 +796,14 @@ export class ChordHost

{ + async #report( + started: readonly StartedFate[], + committed: CommittedReload, + context: Context, + ): Promise { const checks = started.map(async (fate): Promise => { if (fate.kind === "ended" || fate.kind === "restarted") return { object: fate.object, kind: fate.kind }; const cause = await fate.attempt.checked; @@ -601,7 +814,11 @@ export class ChordHost

, facets: ReadonlyMap>): void { - for (const name of names) { - const registries: string[] = []; - for (const [registry, { plugin, registry: token, provider }] of staged.registries) { - if (plugin !== name) continue; + #publish(staged: Staged, facets: ReadonlyMap>): void { + for (const [name, facet] of facets) { + const plugin = staged.plugins.get(name)!; + for (const [registry, { registry: token, provider }] of plugin.registries) { this.registryRecord(registry).serve(token.version, provider); - registries.push(registry); } - const contributions = (staged.contributions.get(name) ?? []).map(({ registry, value }) => + const contributions = plugin.contributions.map(({ registry, value }) => contributionOf( this.registryRecord(registry.name), { plugin: name, type: undefined }, @@ -1205,7 +1420,7 @@ export class ChordHost

boolean, co /** What the report calls a fate. */ const STARTED = { swap: "swapped", restart: "restarted", retry: "retried" } as const; -/** The `aborted` error of a reload whose context aborted: before its commit nothing changed; after it, only its wait. */ +/** The `aborted` error of a reload whose context aborted before its commit: nothing changed. */ function reloadAborted(): ChordError { return new ChordError({ code: "aborted", message: "reload: aborted" }); } -/** The facets of a plugin set by plugin name. */ -function facetsByName

(facets: readonly PluginFacet

[]): Map> { - return new Map(facets.map(({ name, facet }) => [name, facet])); +/** The error of a plugin name given twice: the second is refused. */ +function loadedTwice(name: string): Error { + return new Error(`plugin "${name}" is loaded twice`); +} + +/** Whether registry provider `served` serves readers of `version`: its own, or an older one its `accepts` maps back. */ +function registryServes(served: StagedRegistry, version: number): boolean { + const provided = served.registry.version; + return provided === version || (provided > version && served.provider.accepts?.[version]?.value !== undefined); } diff --git a/packages/chord/src/next/host/index.ts b/packages/chord/src/next/host/index.ts index 79b8a2634..f98bf5b40 100644 --- a/packages/chord/src/next/host/index.ts +++ b/packages/chord/src/next/host/index.ts @@ -7,13 +7,19 @@ export type { Ctx } from "./ctx.ts"; export { defineFacet, type Facet, type FacetHost, type FacetSelf, type ServeOptions } from "./facet.ts"; export type { FollowAttach, FollowAttachment, FollowObserver } from "./follow.ts"; export { + type CommittedReload, createHost, type GetOptions, type Host, type HostError, type HostOptions, type LinkOptions, + type LoadReport, + type PluginFailure, type Position, + type ReloadedObject, + type ReloadOptions, + type ReloadReport, } from "./host.ts"; export type { Native, NativeCloseReason, NativeOptions } from "./native.ts"; export type { Presence } from "./presence.ts"; @@ -24,3 +30,4 @@ export type { RootOptions } from "./root.ts"; export type { RouteOptions } from "./route.ts"; export type { OnHandler, OnLiveHandler, OnNativeHandler, RoutineDeclarations } from "./routines.ts"; export { sleep, untilAborted } from "./sleep.ts"; +export type { PluginFacet } from "./staging.ts"; diff --git a/packages/chord/src/next/host/staging.ts b/packages/chord/src/next/host/staging.ts index 0d61c1977..e26da36a4 100644 --- a/packages/chord/src/next/host/staging.ts +++ b/packages/chord/src/next/host/staging.ts @@ -1,12 +1,14 @@ -// Staging (slice 7b §2): a plugin set's setups run into a staged table beside the live one (the types it serves with -// their providers, the registries it serves, its facet contributions by plugin) with `load`'s checks; and an object's -// new factory run against a recording `self`, which keeps what the factory declares (natives, documents, tables, jobs, -// contributions, requests) and activates, binds and opens nothing, so a reload decides each object's fate before its -// commit. `load` stages its facets the same way and publishes them. +// Staging (slice 7b §2; per plugin, decision "Per plugin at startup and reload"): each plugin's setup runs into a table of +// its own (the types it serves with their providers, the registries it serves, its facet contributions) with `load`'s +// checks, merged into the staged table beside the live one, where a type or registry another plugin serves already +// fails the plugin; and an object's new factory run against a recording `self`, which keeps what the factory declares +// (natives, documents, tables, jobs, contributions, requests with their contract versions) and activates, binds and +// opens nothing, so a reload decides each object's fate, and which plugins keep their old code, before its commit. +// `load` stages its facets the same way and publishes them. import type { Context } from "../../context/index.ts"; import type { Watchable } from "../../watch.ts"; -import type { Contract } from "../contracts/index.ts"; +import type { Contract, ParamsOf } from "../contracts/index.ts"; import { contractData } from "../contracts/internal.ts"; import type { Principal } from "../data/index.ts"; import { bindProvider, checkParams } from "../data/internal.ts"; @@ -33,14 +35,22 @@ export type StagedRegistry = { readonly provider: RegistryProvider; }; -/** A plugin set's setups, run and checked: what it serves and contributes, nothing published yet. */ -export type Staged = { +/** One plugin's setup, run and checked: what it serves and contributes, nothing published yet. */ +export type StagedPlugin = { + readonly name: string; /** The served types by name, each with its plugin, factory, provider (its `accepts` checked) and `linger`. */ - readonly types: Map; + readonly types: ReadonlyMap; /** The served registries by name. */ + readonly registries: ReadonlyMap; + /** The facet contributions, in the setup's order. */ + readonly contributions: readonly FacetContribution[]; +}; + +/** A plugin set's staged plugins, merged: what they serve, by name, and each plugin's own table. */ +export type Staged = { + readonly types: Map; readonly registries: Map; - /** The facet contributions by plugin, in each setup's order. */ - readonly contributions: Map; + readonly plugins: Map; }; /** What staging needs of its host: whom it acts as, where it lives, and its presence lists for setups. */ @@ -53,42 +63,75 @@ export type StagingHost = { /** An npm package name, scoped or not. */ const PLUGIN_NAME = /^(?:@[a-z0-9~-][a-z0-9._~-]*\/)?[a-z0-9~-][a-z0-9._~-]*$/; -/** - * Runs every facet's setup and checks what they declared: plugin names (npm names, each once), a type or registry - * served twice, a remote type on a host without a principal or address, every provider's `accepts` (`checkAccepts`) - * and every registry provider's shape. Throws the first error (naming its plugin where the check does not). - */ -export function stagePlugins

(facets: readonly PluginFacet

[], host: StagingHost): Staged { - const staged: Staged = { types: new Map(), registries: new Map(), contributions: new Map() }; - for (const { name, facet } of facets) { - if (!PLUGIN_NAME.test(name)) throw new Error(`"${name}" is not a valid plugin name`); - if (staged.contributions.has(name)) throw new Error(`plugin "${name}" is loaded twice`); - // The host's presence lists are typed for its `P`. - const references = host.references as (Type: C) => Watchable[]>; - let setup: ReturnType>; - try { - setup = setUp(name, facet, references); - } catch (error) { - throw named(error, `plugin "${name}"`); - } - const { served, contributions } = setup; - for (const entry of served) stageEntry(name, entry, staged, host); - staged.contributions.set(name, contributions); - } - return staged; +/** Whether `name` is a valid plugin name: an npm package name, scoped or not (the plugins component checks it too). */ +export function isPluginName(name: string): boolean { + return PLUGIN_NAME.test(name); } -/** Stages one served entry of `plugin`: a registry's provider, or a type as `Served`. */ -function stageEntry(plugin: string, entry: ServedEntry, staged: Staged, host: StagingHost): void { +/** A staged table with nothing in it. */ +export function emptyStaged(): Staged { + return { types: new Map(), registries: new Map(), plugins: new Map() }; +} + +/** + * Runs one plugin's setup and checks what it declared: its name (an npm name), a type or registry it serves twice, a + * remote type on a host without a principal or address, every provider's `accepts` (`checkAccepts`) and every registry + * provider's shape. Throws the first error, naming the plugin. + */ +export function stagePlugin

({ name, facet }: PluginFacet

, host: StagingHost): StagedPlugin { + if (!isPluginName(name)) throw new Error(`"${name}" is not a valid plugin name`); + // The host's presence lists are typed for its `P`. + const references = host.references as (Type: C) => Watchable[]>; + const types = new Map(); + const registries = new Map(); + try { + const { served, contributions } = setUp(name, facet, references); + for (const entry of served) stageEntry(name, entry, types, registries, host); + return { name, types, registries, contributions }; + } catch (error) { + throw named(error, `plugin "${name}"`); + } +} + +/** + * Merges `plugin` into `staged`; a type or registry another plugin serves there already refuses it (its error + * returned, nothing merged). + */ +export function addStaged(staged: Staged, plugin: StagedPlugin): Error | undefined { + for (const name of plugin.types.keys()) { + const other = staged.types.get(name); + if (other !== undefined) + return new Error(`plugin "${plugin.name}": ${name} is served twice (plugin "${other.plugin}")`); + } + for (const name of plugin.registries.keys()) { + const other = staged.registries.get(name); + if (other !== undefined) { + return new Error(`plugin "${plugin.name}": registry ${name} is served twice (plugin "${other.plugin}")`); + } + } + for (const [name, served] of plugin.types) staged.types.set(name, served); + for (const [name, registry] of plugin.registries) staged.registries.set(name, registry); + staged.plugins.set(plugin.name, plugin); + return undefined; +} + +/** Stages one served entry of `plugin` into its tables: a registry's provider, or a type as `Served`. */ +function stageEntry( + plugin: string, + entry: ServedEntry, + types: Map, + registries: Map, + host: StagingHost, +): void { if (entry.kind === "registry") { const { registry, provider } = entry; - if (staged.registries.has(registry.name)) throw new Error(`registry ${registry.name} is served twice`); + if (registries.has(registry.name)) throw new Error(`registry ${registry.name} is served twice`); checkProvider(registry.name, provider); - staged.registries.set(registry.name, { plugin, registry, provider }); + registries.set(registry.name, { plugin, registry, provider }); return; } const { contract, factory, options } = entry; - if (staged.types.has(contract.name)) throw new Error(`${contract.name} is served twice`); + if (types.has(contract.name)) throw new Error(`${contract.name} is served twice`); // A client serves only local types (§3): a remote object is owned by a principal of the host serving it. if (contract.kind === "remote" && host.principal === undefined) { throw new Error(`${contract.name}: a host without a principal serves only local types`); @@ -115,7 +158,7 @@ function stageEntry(plugin: string, entry: ServedEntry, staged: Staged, host: St contract: contract as RemoteContract, factory: factory as Extract["factory"], }; - staged.types.set(contract.name, served); + types.set(contract.name, served); } /** What a recorded factory declared: what a reload's fate and checks need, nothing activated. */ @@ -126,10 +169,15 @@ export type Recorded = { readonly tables: readonly TableDefinition[]; readonly jobs: readonly JobDefinition[]; readonly contributions: readonly Registry[]; - /** Its requests: the types it gets (`self.get`) and the registries it reads (`self.registry`), by name. */ - readonly requests: readonly string[]; + /** Its requests: the types it gets (`self.get`, with their checked params) and the registries it reads (`self.registry`). */ + readonly requests: readonly Request[]; }; +/** A recorded request, with the contract version it was made at: a type and its params, or a registry. */ +export type Request = + | { readonly kind: "type"; readonly contract: Contract; readonly params: ParamsOf } + | { readonly kind: "registry"; readonly registry: Registry }; + /** * Runs `served`'s factory for `object` (its params and principal) against a recording `self` and returns what it * declared. Every declaration check an attempt makes runs, without effects (declared twice; persisted data in a type @@ -180,7 +228,7 @@ class RecordingSelf { readonly #tables = new Map(); readonly #jobs = new Map>(); readonly #contributions = new Map>(); - readonly #requests: string[] = []; + readonly #requests: Request[] = []; constructor(object: ChordObject, served: Served, context: Context) { this.#object = object; @@ -228,14 +276,14 @@ class RecordingSelf { get(Type: Contract, params: unknown): never { this.#assertDeclaring("get"); - checkParams(Type, params, this.#context); - this.#requests.push(Type.name); + const checked = checkParams(Type, params, this.#context); + this.#requests.push({ kind: "type", contract: Type, params: checked.params }); return INERT; } registry(R: Registry): RegistryValueSource { this.#assertDeclaring("registry"); - this.#requests.push(R.name); + this.#requests.push({ kind: "registry", registry: R }); // The host's own value (an attempt declares the same one): `retryOn` checks its kind. return this.#object.host.registryRecord(R.name).value(R.version); } diff --git a/packages/chord/test/next-node/fixtures/esm-host.ts b/packages/chord/test/next-node/fixtures/esm-host.ts new file mode 100644 index 000000000..5d80be5a8 --- /dev/null +++ b/packages/chord/test/next-node/fixtures/esm-host.ts @@ -0,0 +1,7 @@ +// Fixture (slice 7c): an ESM host package, given to evaluation as its namespace. + +/** The default export a default import reads. */ +export default "esm default"; + +/** A named export a named import reads. */ +export const named = "esm named"; diff --git a/packages/chord/test/next-node/fixtures/plugins/contracts/contract.ts b/packages/chord/test/next-node/fixtures/plugins/contracts/contract.ts new file mode 100644 index 000000000..36a47c01f --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/contracts/contract.ts @@ -0,0 +1,20 @@ +// Fixture plugin (slice 7c): contracts only, no facet; the other fixtures import (copy) it. + +import { hostKind, int, local, method, remote, str } from "@earendil-works/chord/next"; + +/** The host kind of the fixtures' worker. */ +export const Worker = hostKind("test.worker"); + +/** Notes on the worker: a label, a count and `add`. */ +export const Notes = remote("test.notes", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { main: { state: { label: str(), count: int() }, methods: { add: method(undefined, int()) } } }, +}); + +/** The notes' label on the server. */ +export const Index = local("test.index", { version: 1, params: {}, members: { label: str() } }); + +/** A file of a plugin read through `pluginFile`. */ +export const Files = local("test.files", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next-node/fixtures/plugins/contracts/package.json b/packages/chord/test/next-node/fixtures/plugins/contracts/package.json new file mode 100644 index 000000000..8d98413fa --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/contracts/package.json @@ -0,0 +1,5 @@ +{ + "name": "@test/contracts", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next-node/fixtures/plugins/files/assets/data.txt b/packages/chord/test/next-node/fixtures/plugins/files/assets/data.txt new file mode 100644 index 000000000..e749bcd14 --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/files/assets/data.txt @@ -0,0 +1 @@ +a file of the plugin diff --git a/packages/chord/test/next-node/fixtures/plugins/files/package.json b/packages/chord/test/next-node/fixtures/plugins/files/package.json new file mode 100644 index 000000000..d872b0023 --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/files/package.json @@ -0,0 +1,5 @@ +{ + "name": "@test/files", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next-node/fixtures/plugins/files/worker.ts b/packages/chord/test/next-node/fixtures/plugins/files/worker.ts new file mode 100644 index 000000000..1d496441c --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/files/worker.ts @@ -0,0 +1,10 @@ +// Fixture plugin (slice 7c): a facet reading a file of its own plugin through `pluginFile`. + +import { readFileSync } from "node:fs"; +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { pluginFile } from "@earendil-works/chord/next/plugins"; +import { Files } from "../contracts/contract.ts"; + +export default defineFacet((host) => + host.serve(Files, () => ({ text: constant(readFileSync(pluginFile("assets/data.txt"), "utf8").trim()) })), +); diff --git a/packages/chord/test/next-node/fixtures/plugins/notes/label.ts b/packages/chord/test/next-node/fixtures/plugins/notes/label.ts new file mode 100644 index 000000000..a380265dd --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// Fixture plugin (slice 7c): the label both facets show; tests edit it. + +export const label = "v1"; diff --git a/packages/chord/test/next-node/fixtures/plugins/notes/package.json b/packages/chord/test/next-node/fixtures/plugins/notes/package.json new file mode 100644 index 000000000..d52261420 --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@test/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next-node/fixtures/plugins/notes/server.ts b/packages/chord/test/next-node/fixtures/plugins/notes/server.ts new file mode 100644 index 000000000..dc4429bef --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/notes/server.ts @@ -0,0 +1,7 @@ +// Fixture plugin (slice 7c): the server facet, serving `Index` with the same label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Index } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => host.serve(Index, () => ({ label: constant(label) }))); diff --git a/packages/chord/test/next-node/fixtures/plugins/notes/worker.ts b/packages/chord/test/next-node/fixtures/plugins/notes/worker.ts new file mode 100644 index 000000000..c128c7b8a --- /dev/null +++ b/packages/chord/test/next-node/fixtures/plugins/notes/worker.ts @@ -0,0 +1,26 @@ +// Fixture plugin (slice 7c): the worker facet, serving `Notes` with a count document. + +import { constant, defineFacet, defineState, int, obj } from "@earendil-works/chord/next"; +import { Notes } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +export default defineFacet((host) => + host.serve(Notes, (_params, _principal, self) => { + const doc = self.state(Count); + return { + variant: "main", + members: { + label: constant(label), + count: self.derive([doc], ([value]) => value.n), + add: async (ctx) => { + await doc.change((draft) => { + draft.n += 1; + }, ctx); + return doc.value.n; + }, + }, + }; + }), +); diff --git a/packages/chord/test/next-node/plugins/build.test.ts b/packages/chord/test/next-node/plugins/build.test.ts new file mode 100644 index 000000000..623e85873 --- /dev/null +++ b/packages/chord/test/next-node/plugins/build.test.ts @@ -0,0 +1,256 @@ +// Tests of building (slice 7c §2): every row of the import table, failures as results, source maps relative to the +// plugin and digests stable across two checkout paths, inputs hashed as esbuild read them. + +import { createHash } from "node:crypto"; +import { readFile, realpath } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + type BuildOptions, + type BuiltFacet, + buildFacet, + discoverPlugins, + type Plugin, +} from "../../../src/next-node/plugins/index.ts"; +import { KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +const NODE: BuildOptions = { hostPackages: ["@earendil-works/chord/next"], platform: "node" }; +const BROWSER: BuildOptions = { + hostPackages: ["@earendil-works/chord/next", "react/jsx-runtime"], + platform: "browser", + publicPath: "/assets/", +}; + +/** The plugins under `root`, and the one named `name`. */ +async function discovered(root: string, name: string): Promise<{ plugins: readonly Plugin[]; plugin: Plugin }> { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + return { plugins, plugin: plugins.find((each) => each.name === name)! }; +} + +/** `built`'s script, failing the test with its errors when it failed. */ +function script(built: BuiltFacet): Extract { + if (built.kind === "failed") throw new Error(`build failed: ${built.errors.join("\n")}`); + return built; +} + +/** The errors of a build that must fail. */ +function errors(built: BuiltFacet): readonly string[] { + if (built.kind !== "failed") throw new Error("the build succeeded"); + return built.errors; +} + +/** The sources of `built`'s inline source map. */ +function sourcesOf(built: Extract): string[] { + const inline = /sourceMappingURL=data:application\/json;base64,(\S+)/.exec(built.source)![1]!; + return JSON.parse(Buffer.from(inline, "base64").toString()).sources; +} + +describe("building a facet: the import table", () => { + it("inlines the plugin's own files and another plugin's file (copied); keeps host packages external", async () => { + const root = await tempRoot(["contracts", "notes"]); + const { plugins, plugin } = await discovered(root, "@test/notes"); + const built = script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + const real = await realpath(root); + expect(built).toMatchObject({ plugin: "@test/notes", facet: "worker", assets: [] }); + expect(built.source.startsWith("(function (require, module, exports) {")).toBe(true); + expect(built.source).toContain('require("@earendil-works/chord/next")'); + expect(built.copied).toEqual([join(real, "contracts", "contract.ts")]); + expect(built.inputs.map(({ file }) => file)).toEqual([ + join(real, "contracts", "contract.ts"), + join(real, "notes", "label.ts"), + join(real, "notes", "worker.ts"), + ]); + // Relative to the plugin (a file whose code minified away maps nothing and is left out). + expect(sourcesOf(built)).toEqual(expect.arrayContaining(["worker.ts", "../contracts/contract.ts"])); + for (const source of sourcesOf(built)) expect(source.startsWith("/")).toBe(false); + // Positions only: no copy of the sources' text in the script (Opus N3). + const inline = /sourceMappingURL=data:application\/json;base64,(\S+)/.exec(built.source)![1]!; + expect(JSON.parse(Buffer.from(inline, "base64").toString())).not.toHaveProperty("sourcesContent"); + }); + + it("hashes each input as esbuild read it; the digest is the source's hash", async () => { + const root = await tempRoot(["contracts", "notes"]); + const { plugins, plugin } = await discovered(root, "@test/notes"); + const built = script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + for (const { file, hash } of built.inputs) { + expect(hash).toBe( + createHash("sha256") + .update(await readFile(file)) + .digest("hex"), + ); + } + expect(built.digest).toBe(createHash("sha256").update(built.source).digest("hex")); + }); + + it("gives the same script and digest from two checkout paths (relative source maps)", async () => { + const first = await tempRoot(["contracts", "notes"]); + const second = await tempRoot(["contracts", "notes"]); + const builds = await Promise.all( + [first, second].map(async (root) => { + const { plugins, plugin } = await discovered(root, "@test/notes"); + return script(await buildFacet(plugin, "worker", { ...NODE, plugins }, context)); + }), + ); + expect(builds[0]!.digest).toBe(builds[1]!.digest); + expect(builds[0]!.source).toBe(builds[1]!.source); + }); + + it("inlines a package of its dependencies; keeps chord.external packages and Node builtins external on Node", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "deps", + "@test/deps", + { + "worker.ts": [ + 'import { pad } from "pad-dep";', + 'import native from "native-dep";', + 'import { join } from "node:path";', + 'import { readFileSync } from "fs";', + 'import { pluginFile } from "@earendil-works/chord/next/plugins";', + "export default { setup: () => [pad, native, join, readFileSync, pluginFile] };", + ].join("\n"), + }, + { dependencies: { "pad-dep": "1.0.0" }, chord: { external: ["native-dep"] } }, + ); + await writePackage(dir, "pad-dep", { "index.js": 'exports.pad = (s) => "pad:" + s;' }); + await writePackage(dir, "native-dep", { "index.js": "module.exports = 1;" }); + const { plugin } = await discovered(root, "@test/deps"); + const built = script(await buildFacet(plugin, "worker", NODE, context)); + expect(built.source).toContain('"pad:"'); + for (const external of ["native-dep", "node:path", "fs", "@earendil-works/chord/next/plugins"]) { + expect(built.source).toContain(`require("${external}")`); + } + // Packages are no inputs: the watcher ignores node_modules. + expect(built.inputs.map(({ file }) => file)).toEqual([join(await realpath(dir), "worker.ts")]); + }); + + it("refuses a chord.external package and a Node builtin in a browser build", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "web", + "@test/web", + { + "web.ts": + 'import native from "native-dep";\nimport { join } from "node:path";\nexport default { setup: () => [native, join] };', + }, + { chord: { external: ["native-dep"] } }, + ); + await writePackage(dir, "native-dep", { "index.js": "module.exports = 1;" }); + const { plugin } = await discovered(root, "@test/web"); + expect(errors(await buildFacet(plugin, "web", BROWSER, context))).toEqual([ + 'web.ts:1:20: "native-dep": chord.external packages are for Node builds', + 'web.ts:2:22: "node:path": a Node builtin in a browser build', + ]); + }); + + it("collects a browser build's CSS into one stylesheet and its assets into content-hashed files", async () => { + const root = await tempRoot(); + await writePlugin(root, "web", "@test/web", { + "web.tsx": [ + 'import "./notes.css";', + 'import iconUrl from "./assets/icon.svg?url";', + "const Icon = () => ;", + "export default { setup: () => Icon };", + ].join("\n"), + "notes.css": ".notes { color: red; }", + "assets/icon.svg": "", + }); + const { plugin } = await discovered(root, "@test/web"); + const built = script(await buildFacet(plugin, "web", BROWSER, context)); + expect(built.css?.name).toMatch(/^web-[A-Z0-9]+\.css$/); + expect(built.css?.text).toContain(".notes{color:red}"); + expect(built.assets.map(({ name }) => name)).toEqual([expect.stringMatching(/^icon-[A-Z0-9]+\.svg$/)]); + expect(new TextDecoder().decode(built.assets[0]!.bytes)).toBe(""); + expect(built.source).toContain(`"/assets/${built.assets[0]!.name}"`); + expect(built.source).toContain('require("react/jsx-runtime")'); + }); + + it("refuses CSS and assets in a Node build", async () => { + const root = await tempRoot(); + await writePlugin(root, "n", "@test/n", { + "worker.ts": 'import "./a.css";\nimport url from "./icon.svg?url";\nexport default { setup: () => url };', + "a.css": "a{}", + "icon.svg": "", + }); + const { plugin } = await discovered(root, "@test/n"); + const found = errors(await buildFacet(plugin, "worker", NODE, context)); + expect(found).toEqual([ + 'worker.ts:1:8: "./a.css": stylesheets and assets are for browser builds', + expect.stringMatching(/^worker\.ts:2:17: "\.\/icon\.svg\?url": assets by URL are for browser builds/), + ]); + }); + + it("refuses import.meta.url, __dirname, __filename, a dynamic import() and top-level await, pointing to pluginFile", async () => { + const root = await tempRoot(); + await writePlugin(root, "bad", "@test/bad", { + "worker.ts": "export const a = import.meta.url;\nexport default { setup: () => [__dirname, __filename] };", + "server.ts": 'export default { setup: () => import("./other.ts") };', + "web.ts": "await Promise.resolve();\nexport default { setup: () => {} };", + "other.ts": "export const x = 1;", + }); + const { plugin } = await discovered(root, "@test/bad"); + const meta = errors(await buildFacet(plugin, "worker", NODE, context)); + expect(meta.length).toBeGreaterThan(0); + for (const error of meta) + expect(error).toMatch(/^worker\.ts:\d+:\d+: import\.meta\.url, __dirname and __filename .*pluginFile/); + // Without `import.meta` in the file: each free `__dirname` and `__filename` at its own line (example 37). + await writeFiles(join(root, "bad"), { + "worker.ts": "// The globals.\nconsole.log(__dirname);\nexport default { setup: () => __filename };", + }); + expect(errors(await buildFacet(plugin, "worker", NODE, context))).toEqual([ + expect.stringMatching(/^worker\.ts:2:13: import\.meta\.url, __dirname and __filename .*pluginFile/), + expect.stringMatching(/^worker\.ts:3:\d+: import\.meta\.url, __dirname and __filename .*pluginFile/), + ]); + expect(errors(await buildFacet(plugin, "server", NODE, context))).toEqual([ + expect.stringMatching( + /^server\.ts:1:\d+: import\("\.\/other\.ts"\): a plugin script has no dynamic import .*pluginFile/, + ), + ]); + expect(errors(await buildFacet(plugin, "web", NODE, context))).toEqual([ + "web.ts:1:1: a plugin script has no top-level await (a facet declares synchronously)", + ]); + }); + + it("refuses a path outside the plugin, a package not in its dependencies, and a package in a copied file", async () => { + const root = await tempRoot(["contracts"]); + await writePlugin(root, "out", "@test/out", { + "worker.ts": + 'import { base } from "../base.ts";\nimport x from "left-out";\nexport default { setup: () => [base, x] };', + }); + await writeFiles(root, { "base.ts": "export const base = 1;" }); + await writePlugin(root, "copier", "@test/copier", { + "worker.ts": 'import { y } from "../shared/shared.ts";\nexport default { setup: () => y };', + }); + await writePlugin(root, "shared", "@test/shared", { + "shared.ts": 'import y from "some-package";\nexport { y };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const out = plugins.find(({ name }) => name === "@test/out")!; + expect(errors(await buildFacet(out, "worker", { ...NODE, plugins }, context))).toEqual([ + 'worker.ts:1:22: "../base.ts": outside the plugin (base code is reached through a host package, packages through dependencies)', + 'worker.ts:2:15: "left-out": neither a host package nor in the dependencies of plugin "@test/out"', + ]); + const copier = plugins.find(({ name }) => name === "@test/copier")!; + expect(errors(await buildFacet(copier, "worker", { ...NODE, plugins }, context))).toEqual([ + '../shared/shared.ts:1:15: "some-package": a file copied from plugin "@test/shared" may import only host packages and files of its own plugin', + ]); + }); + + it("returns a syntax error and a missing facet as failed results, never a throw", async () => { + const root = await tempRoot(); + await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); + const { plugin } = await discovered(root, "@test/syntax"); + expect(errors(await buildFacet(plugin, "worker", NODE, context))).toEqual([ + expect.stringMatching(/^worker\.ts:1:\d+: Expected /), + ]); + expect(await buildFacet(plugin, "server", NODE, context)).toEqual({ + kind: "failed", + plugin: "@test/syntax", + facet: "server", + errors: ['plugin "@test/syntax" has no server facet'], + }); + }); +}); diff --git a/packages/chord/test/next-node/plugins/discover.test.ts b/packages/chord/test/next-node/plugins/discover.test.ts new file mode 100644 index 000000000..741e24658 --- /dev/null +++ b/packages/chord/test/next-node/plugins/discover.test.ts @@ -0,0 +1,95 @@ +// Tests of discovery (slice 7c §1): names from package.json, the first root winning a duplicate, directories reported +// (no or an invalid package.json, an invalid name, two files of one kind), contracts-only plugins, symlinks. + +import { mkdir, realpath, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { discoverPlugins } from "../../../src/next-node/plugins/index.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +describe("discovery", () => { + it("names a plugin by its package.json, lists its facet files by kind, its dependencies and externals", async () => { + const root = await tempRoot(["contracts", "notes"]); + const dir = await writePlugin( + root, + "deps-dir", + "@test/deps", + { "worker.ts": "", "store.ts": "" }, + { + dependencies: { marked: "1.0.0" }, + chord: { external: ["better-sqlite3"] }, + }, + ); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(problems).toEqual([]); + const real = await realpath(root); + expect(plugins).toEqual([ + { name: "@test/contracts", dir: join(real, "contracts"), facets: {}, dependencies: [], external: [] }, + { + name: "@test/deps", + dir: await realpath(dir), + facets: { worker: join(real, "deps-dir", "worker.ts") }, + dependencies: ["marked"], + external: ["better-sqlite3"], + }, + { + name: "@test/notes", + dir: join(real, "notes"), + facets: { server: join(real, "notes", "server.ts"), worker: join(real, "notes", "worker.ts") }, + dependencies: [], + external: [], + }, + ]); + }); + + it("reports a directory without package.json, with an invalid one, an invalid name or two files of a kind", async () => { + const root = await tempRoot(); + await mkdir(join(root, "bare")); + await writeFiles(join(root, "broken"), { "package.json": "{ not json" }); + await writePlugin(root, "invalid", "Not A Name", {}); + await writePlugin(root, "two", "@test/two", { "worker.ts": "", "worker.js": "" }); + await writePlugin(root, "good", "@test/good", { "web.tsx": "" }); + // Not plugins and not reported: hidden directories and a root's own packages. + await mkdir(join(root, ".git")); + await mkdir(join(root, "node_modules")); + const { plugins, problems } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map(({ name }) => name)).toEqual(["@test/good"]); + const real = await realpath(root); + expect(problems).toEqual([ + { path: join(real, "bare"), error: "no package.json" }, + { path: join(real, "broken"), error: expect.stringMatching(/^package\.json: /) }, + { path: join(real, "invalid"), error: 'package.json: "Not A Name" is not a valid plugin name' }, + { path: join(real, "two"), error: "two worker facets: worker.ts, worker.js" }, + ]); + }); + + it("keeps the first root's plugin of a duplicate name and reports the other; a missing root is empty", async () => { + const first = await tempRoot(); + const second = await tempRoot(); + await writePlugin(first, "a", "@test/same", { "worker.ts": "" }); + await writePlugin(second, "b", "@test/same", { "server.ts": "" }); + const { plugins, problems } = await discoverPlugins( + [first, join(first, "missing"), second], + { kinds: KINDS }, + context, + ); + expect(plugins.map(({ dir }) => dir)).toEqual([join(await realpath(first), "a")]); + expect(problems).toEqual([ + { + path: join(await realpath(second), "b"), + error: `plugin "@test/same" is in ${join(await realpath(first), "a")} already`, + }, + ]); + }); + + it("follows a symlinked plugin directory to its real path", async () => { + const elsewhere = await tempRoot(); + const root = await tempRoot(); + const dir = await writePlugin(elsewhere, "linked", "@test/linked", { "worker.ts": "" }); + await symlink(dir, join(root, "link")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + expect(plugins.map((each) => [each.name, each.dir])).toEqual([["@test/linked", await realpath(dir)]]); + expect(plugins[0]!.facets).toEqual({ worker: join(await realpath(dir), "worker.ts") }); + }); +}); diff --git a/packages/chord/test/next-node/plugins/evaluate.test.ts b/packages/chord/test/next-node/plugins/evaluate.test.ts new file mode 100644 index 000000000..bb2c3c990 --- /dev/null +++ b/packages/chord/test/next-node/plugins/evaluate.test.ts @@ -0,0 +1,211 @@ +// Tests of evaluation (slice 7c §3): host packages as the host's instances (`instanceof ChordError` across a plugin), +// default and named imports of ESM and CommonJS host packages, chord.external packages and builtins from the plugin, +// `pluginFile` and its `..` refusal, an import-time throw, a non-facet export, a refused require, a computed import(), +// the filename. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { local, str } from "../../../src/next/contracts/index.ts"; +import { ChordError, isChordError } from "../../../src/next/data/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import { + type BuiltScript, + buildFacet, + discoverPlugins, + evaluateFacet, + type Plugin, + pluginFile, +} from "../../../src/next-node/plugins/index.ts"; +import { expectCollected } from "../../next/support.ts"; +import * as esmHost from "../fixtures/esm-host.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writePackage, writePlugin } from "./support.ts"; + +const HOSTS = { ...HOST_PACKAGES, "esm-host": esmHost, "cjs-host": { named: "cjs named" } }; + +/** The fixtures' `Files` contract, as the `contracts` fixture declares it. */ +const Files = local("test.files", { version: 1, params: {}, members: { text: str() } }); + +/** Builds `name`'s `kind` facet under `root` for Node and returns the script and the plugin. */ +async function built(root: string, name: string, kind = "worker"): Promise<{ script: BuiltScript; plugin: Plugin }> { + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins.find((each) => each.name === name)!; + const result = await buildFacet( + plugin, + kind, + { hostPackages: Object.keys(HOSTS), platform: "node", plugins }, + context, + ); + if (result.kind === "failed") throw new Error(result.errors.join("\n")); + return { script: result, plugin }; +} + +/** A facet with extra members a test reads. */ +type Probe = { readonly setup: () => void; readonly [name: string]: unknown }; + +describe("evaluating a facet", () => { + it("gives host packages as the host's instances: a ChordError crosses the plugin as one", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": [ + 'import { ChordError, defineFacet } from "@earendil-works/chord/next";', + "const facet = defineFacet(() => {});", + 'export default { ...facet, made: new ChordError({ code: "unavailable", message: "m" }), is: (e: unknown) => e instanceof ChordError };', + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.made).toBeInstanceOf(ChordError); + expect(isChordError(facet.made, "unavailable")).toBe(true); + expect((facet.is as (e: unknown) => boolean)(new ChordError({ code: "unavailable", message: "x" }))).toBe(true); + }); + + it("answers default and named imports of an ESM and a CommonJS host package", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": [ + 'import esm, { named as esmNamed } from "esm-host";', + 'import * as esmAll from "esm-host";', + 'import cjs, { named as cjsNamed } from "cjs-host";', + "export default { setup: () => {}, values: [esm, esmNamed, esmAll.named, cjs, cjsNamed] };", + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.values).toEqual(["esm default", "esm named", "esm named", { named: "cjs named" }, "cjs named"]); + }); + + it("loads chord.external packages and Node builtins from the plugin's directory", async () => { + const root = await tempRoot(); + const dir = await writePlugin( + root, + "p", + "@test/p", + { + "worker.ts": + 'import ext from "ext-dep";\nimport { basename } from "node:path";\nexport default { setup: () => {}, values: [ext, basename("/a/b")] };', + }, + { chord: { external: ["ext-dep"] } }, + ); + await writePackage(dir, "ext-dep", { "index.js": 'module.exports = "from the plugin";' }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + expect(facet.values).toEqual(["from the plugin", "b"]); + }); + + it("gives each facet pluginFile for its own plugin's directory and refuses .. out of it", async () => { + const root = await tempRoot(["contracts", "files"]); + const { script, plugin } = await built(root, "@test/files"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }); + const host = createHost({ onError: () => {} }); + await host.load([{ name: "@test/files", facet }], context); + expect((await host.get(Files, {}, context)).text.value).toBe("a file of the plugin"); + await host.close(context); + await writePlugin(root, "escape", "@test/escape", { + "worker.ts": + 'import { pluginFile } from "@earendil-works/chord/next/plugins";\nexport default { setup: () => {}, escape: () => pluginFile("../files/assets/data.txt") };', + }); + const escaping = await built(root, "@test/escape"); + const probe = evaluateFacet(escaping.script, { + hostPackages: HOSTS, + plugin: escaping.plugin, + }) as unknown as Probe; + expect(() => (probe.escape as () => string)()).toThrow( + 'pluginFile("../files/assets/data.txt"): outside the directory of plugin "@test/escape"', + ); + // Outside an evaluated facet it has none. + expect(() => pluginFile("x")).toThrow("only a plugin's facet script"); + }); + + it("throws an import-time throw, a non-facet export and a refused require, naming the facet", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'throw new Error("import time");\nexport default { setup: () => {} };', + "server.ts": "export default 42;", + }); + const thrown = await built(root, "@test/p"); + expect(() => evaluateFacet(thrown.script, { hostPackages: HOSTS, plugin: thrown.plugin })).toThrow("import time"); + const notFacet = await built(root, "@test/p", "server"); + expect(() => evaluateFacet(notFacet.script, { hostPackages: HOSTS, plugin: notFacet.plugin })).toThrow( + 'plugin "@test/p" (server): its default export is not a facet (defineFacet)', + ); + const handmade: BuiltScript = { + ...thrown.script, + source: '(function (require, module, exports) { require("nope"); })', + }; + expect(() => evaluateFacet(handmade, { hostPackages: HOSTS, plugin: thrown.plugin })).toThrow( + 'plugin "@test/p" (worker): "nope" is not a host package, a Node builtin or one of its chord.external packages', + ); + }); + + it("runs a computed import() through the facet's require: an unknown one names the facet (Astra)", async () => { + const root = await tempRoot(); + await writePlugin(root, "dynamic", "@test/dynamic", { + "worker.ts": [ + 'import { defineFacet } from "@earendil-works/chord/next";', + "const later = (specifier: string) => import(specifier);", + "export default { ...defineFacet(() => {}), later };", + ].join("\n"), + }); + const { script, plugin } = await built(root, "@test/dynamic"); + // The fixture attaches its computed import to the facet. + const { later } = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as { + later: (specifier: string) => Promise<{ readonly named?: string }>; + }; + await expect(later("missing-package")).rejects.toThrow( + 'plugin "@test/dynamic" (worker): "missing-package" is not a host package', + ); + expect((await later("cjs-host")).named).toBe("cjs named"); + }); + + it("runs a script under a stable filename per plugin, kind and digest", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'export default { setup: () => {}, fail: () => { throw new Error("x"); } };', + }); + const { script, plugin } = await built(root, "@test/p"); + const facet = evaluateFacet(script, { hostPackages: HOSTS, plugin }) as unknown as Probe; + let stack = ""; + try { + (facet.fail as () => void)(); + } catch (error) { + stack = (error as Error).stack!; + } + expect(stack).toContain(`/chord-plugins/@test/p/worker.ts?${script.digest}:`); + }); +}); + +describe("ESM host packages' wrappers", () => { + it("wraps a namespace once, and keeps no namespace nobody else holds", async () => { + const root = await tempRoot(); + await writePlugin(root, "p", "@test/p", { + "worker.ts": 'import * as ns from "fake-esm";\nexport default { setup: () => {}, ns };', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const result = await buildFacet(plugin, "worker", { hostPackages: ["fake-esm"], platform: "node" }, context); + if (result.kind === "failed") throw new Error(result.errors.join("\n")); + // What `Object.prototype.toString` calls a module namespace. + const evaluate = (namespace: object) => + evaluateFacet(result, { hostPackages: { "fake-esm": namespace }, plugin }) as unknown as Probe; + // Wrapping reads each export once: twice evaluated, read once. + let reads = 0; + const namespace = { + [Symbol.toStringTag]: "Module", + get named() { + reads++; + return 1; + }, + }; + expect([ + (evaluate(namespace).ns as { named: number }).named, + (evaluate(namespace).ns as { named: number }).named, + ]).toEqual([1, 1]); + expect(reads).toBe(1); + const ref = (() => { + const dropped = { [Symbol.toStringTag]: "Module", named: 2 }; + expect((evaluate(dropped).ns as { named: number }).named).toBe(2); + return new WeakRef(dropped); + })(); + await expectCollected(ref); + }); +}); diff --git a/packages/chord/test/next-node/plugins/install.test.ts b/packages/chord/test/next-node/plugins/install.test.ts new file mode 100644 index 000000000..c7e56f1f5 --- /dev/null +++ b/packages/chord/test/next-node/plugins/install.test.ts @@ -0,0 +1,62 @@ +// Tests of installing a plugin's dependencies (slice 7c §5): the npm arguments, run in the plugin's real directory, +// the command overridable; a failure and an abort reject. + +import { readFile, realpath, symlink } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context, withCancel } from "../../../src/context/index.ts"; +import { discoverPlugins, installDependencies } from "../../../src/next-node/plugins/index.ts"; +import { rejection } from "../../next/support.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** A fake npm (a Node script) that records its arguments and directory, and exits with `code` after `delayMs`. */ +async function fakeNpm(dir: string, code: number, delayMs = 0): Promise { + const script = join(dir, "fake-npm.mjs"); + await writeFiles(dir, { + "fake-npm.mjs": [ + 'import { writeFileSync } from "node:fs";', + `writeFileSync(${JSON.stringify(join(dir, "record.json"))}, JSON.stringify({ args: process.argv.slice(2), cwd: process.cwd() }));`, + `setTimeout(() => { if (${code} !== 0) console.error("npm said no"); process.exit(${code}); }, ${delayMs});`, + ].join("\n"), + }); + return [process.execPath, script]; +} + +describe("installDependencies", () => { + it("runs npm install without dev dependencies, peers, scripts, audit or funding, in the plugin's real directory", async () => { + const tools = await tempRoot(); + const elsewhere = await tempRoot(); + const root = await tempRoot(); + const dir = await writePlugin( + elsewhere, + "deps", + "@test/deps", + { "worker.ts": "" }, + { dependencies: { a: "1.0.0" } }, + ); + await symlink(dir, join(root, "deps")); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + await installDependencies(plugins[0]!, context, { npmCommand: await fakeNpm(tools, 0) }); + expect(JSON.parse(await readFile(join(tools, "record.json"), "utf8"))).toEqual({ + args: ["install", "--omit=dev", "--legacy-peer-deps", "--ignore-scripts", "--no-audit", "--no-fund"], + cwd: await realpath(dir), + }); + }); + + it("rejects with the command's output when it fails, and when the context aborts", async () => { + const tools = await tempRoot(); + const root = await tempRoot(); + await writePlugin(root, "deps", "@test/deps", {}); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + expect( + await rejection(installDependencies(plugin, context, { npmCommand: await fakeNpm(tools, 1) })), + ).toMatchObject({ + message: expect.stringMatching(/^plugin "@test\/deps": .* install failed \(exit 1\): npm said no$/), + }); + const { context: aborted, cancel } = withCancel(context); + const running = installDependencies(plugin, aborted, { npmCommand: await fakeNpm(tools, 0, 5_000) }); + setTimeout(() => cancel(new Error("stop")), 50); + expect(await rejection(running)).toMatchObject({ name: "AbortError" }); + }); +}); diff --git a/packages/chord/test/next-node/plugins/plugins.test.ts b/packages/chord/test/next-node/plugins/plugins.test.ts new file mode 100644 index 000000000..b9c435d10 --- /dev/null +++ b/packages/chord/test/next-node/plugins/plugins.test.ts @@ -0,0 +1,585 @@ +// Tests of the plugins component (slice 7c §4, §5, §7): per plugin at startup and reload (a failing build, evaluation, +// setup; unchanged plugins untouched; added and removed plugins; the closure rule through builds), `rollout` (order, +// stopping at a rejecting step, marks), installs through the component, GC over 200 reloads, and end to end a server +// and a worker loading plugin directories, an edit rolled out, a broken edit kept old on both hosts. + +import { rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context, withAbortSignal } from "../../../src/context/index.ts"; +import { hostKind, int, local, method, remote, str } from "../../../src/next/contracts/index.ts"; +import { createHost, defineFacet, type Host, type ReloadReport } from "../../../src/next/host/index.ts"; +import type { ChordHost } from "../../../src/next/host/internal.ts"; +import { memoryLinkPair } from "../../../src/next/link/index.ts"; +import { constant } from "../../../src/next/state/index.ts"; +import { createPlugins, type Plugins, type RolloutStep } from "../../../src/next-node/plugins/index.ts"; +import { + changesOf, + delivered, + expectCollected, + rejection, + settledHeap, + transported, + valuesOf, +} from "../../next/support.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writeFiles, writePackage, writePlugin } from "./support.ts"; + +/** The fixtures' contracts, as the `contracts` fixture declares them. */ +const Worker = hostKind("test.worker"); +const Notes = remote("test.notes", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { main: { state: { label: str(), count: int() }, methods: { add: method(undefined, int()) } } }, +}); +const Index = local("test.index", { version: 1, params: {}, members: { label: str() } }); +const Item = local("test.item", { version: 1, params: {}, members: { label: str() } }); + +/** A plugin `dir` named `@test/

` serving `Item` labelled `label` from its worker facet. */ +async function itemPlugin(root: string, dir: string, label: string): Promise { + await writePlugin(root, dir, `@test/${dir}`, { + "worker.ts": [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + `const Item = local("test.${dir}", { version: 1, params: {}, members: { label: str() } });`, + `export default defineFacet((host) => host.serve(Item, () => ({ label: constant(${JSON.stringify(label)}) })));`, + ].join("\n"), + }); +} + +/** The component over `root`, its reports collected. */ +function component(root: string): { plugins: Plugins; reported: unknown[] } { + const reported: unknown[] = []; + const plugins = createPlugins( + { + roots: [root], + kinds: KINDS, + browserKinds: ["web"], + hostPackages: HOST_PACKAGES, + onError: (e) => reported.push(e), + }, + context, + ); + return { plugins, reported }; +} + +/** The failures of a report as plugin names. */ +const failedNames = (report: { readonly failed: readonly { readonly plugin: string }[] }) => + report.failed.map(({ plugin }) => plugin); + +describe("per plugin at startup", () => { + it("leaves out a plugin whose build fails, whose evaluation throws or whose setup throws; loads the others and core", async () => { + const root = await tempRoot(); + await itemPlugin(root, "good", "good"); + await writePlugin(root, "syntax", "@test/syntax", { "worker.ts": "export default {" }); + await writePlugin(root, "throws", "@test/throws", { + "worker.ts": 'throw new Error("at import");\nexport default {};', + }); + await writePlugin(root, "setup", "@test/setup", { + "worker.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => { throw new Error("in setup"); });', + }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + const core = defineFacet((h) => h.serve(Item, () => ({ label: constant("core") }))); + const report = await plugins.load(host, { kind: "worker", core: [{ name: "@app/core", facet: core }] }, context); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + [ + "@test/syntax", + expect.stringMatching(/^plugin "@test\/syntax": its worker facet failed to build:\nworker\.ts:1:/), + ], + ["@test/throws", "at import"], + ["@test/setup", 'plugin "@test/setup": in setup'], + ]); + const Good = local("test.good", { version: 1, params: {}, members: { label: str() } }); + expect((await host.get(Good, {}, context)).label.value).toBe("good"); + expect((await host.get(Item, {}, context)).label.value).toBe("core"); + await host.close(context); + plugins.close(); + }); +}); + +describe("per plugin at reload", () => { + /** A host loaded with plugins `a`, `b` and `c` (worker facets), and the component. */ + async function loaded() { + const root = await tempRoot(); + for (const dir of ["a", "b", "c"]) await itemPlugin(root, dir, `${dir}1`); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + const label = async (dir: string) => + (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) + .label.value; + return { root, plugins, host, label }; + } + + it("rebuilds one plugin of three: only it changes; the others are passed as the same facets, untouched", async () => { + const { root, plugins, host, label } = await loaded(); + await itemPlugin(root, "b", "b2"); + const built = await plugins.build(["@test/b"], context); + expect(built.facets.map(({ plugin, facet }) => [plugin, facet])).toEqual([["@test/b", "worker"]]); + const report = await plugins.reload(host, built, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/b"], removed: [] }); + expect(report.objects).toEqual([{ object: "test.b", kind: "swapped" }]); + expect([await label("a"), await label("b"), await label("c")]).toEqual(["a1", "b2", "c1"]); + // Built again unchanged: the same digest, nothing reloaded. + const again = await plugins.reload(host, await plugins.build(["@test/b"], context), context); + expect(again.plugins).toEqual({ added: [], changed: [], removed: [] }); + await host.close(context); + plugins.close(); + }); + + it("keeps the loaded facet of a plugin whose build fails or whose evaluation throws; the others apply", async () => { + const { root, plugins, host, label } = await loaded(); + await writeFiles(join(root, "a"), { "worker.ts": "export default {" }); + await writeFiles(join(root, "b"), { "worker.ts": 'throw new Error("at import");\nexport default {};' }); + await itemPlugin(root, "c", "c2"); + const report = await plugins.reload( + host, + await plugins.build(["@test/a", "@test/b", "@test/c"], context), + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/c"], removed: [] }); + expect(failedNames(report)).toEqual(["@test/a", "@test/b"]); + expect([await label("a"), await label("b"), await label("c")]).toEqual(["a1", "b1", "c2"]); + await host.close(context); + plugins.close(); + }); + + it("adds a plugin discovered since, removes one no longer there; an added plugin that fails is left out", async () => { + const { root, plugins, host, label } = await loaded(); + await itemPlugin(root, "d", "d1"); + await writePlugin(root, "e", "@test/e", { "worker.ts": "export default {" }); + await rm(join(root, "c"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const built = await plugins.build([], context); + expect(built.removed).toEqual(["@test/c"]); + const report = await plugins.reload(host, built, context); + expect(report.plugins).toEqual({ added: ["@test/d"], changed: [], removed: ["@test/c"] }); + expect(failedNames(report)).toEqual(["@test/e"]); + expect(await label("d")).toBe("d1"); + await host.close(context); + plugins.close(); + }); + + it("keeps a consumer edited together with a provider whose new build fails (the closure rule through builds)", async () => { + const root = await tempRoot(); + const provider = (version: number) => + [ + 'import { constant, defineFacet, local, int } from "@earendil-works/chord/next";', + `export const P = local("test.provider", { version: ${version}, params: {}, members: { n: int() } });`, + `export default defineFacet((host) => host.serve(P, () => ({ n: constant(${version}) })));`, + ].join("\n"); + const consumer = (version: number) => + [ + 'import { defineFacet, local, int, str } from "@earendil-works/chord/next";', + `const P = local("test.provider", { version: ${version}, params: {}, members: { n: int() } });`, + 'const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } });', + `export default defineFacet((host) => host.serve(C, (_, self) => ({ label: self.derive([self.get(P, {}).n], ([n]) => "v${version} " + n) })));`, + ].join("\n"); + await writePlugin(root, "provider", "@test/provider", { "worker.ts": provider(1) }); + await writePlugin(root, "consumer", "@test/consumer", { "worker.ts": consumer(1) }); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + const C = local("test.consumer", { version: 1, params: {}, members: { label: str() } }); + const shown = await host.get(C, {}, context); + await writeFiles(join(root, "provider"), { "worker.ts": `${provider(2)}\nexport const broken = ;` }); + await writeFiles(join(root, "consumer"), { "worker.ts": consumer(2) }); + const report = await plugins.reload( + host, + await plugins.build(["@test/provider", "@test/consumer"], context), + context, + ); + expect(failedNames(report)).toEqual(["@test/provider", "@test/consumer"]); + expect((report.failed[1]!.error as Error).message).toBe( + 'plugin "@test/consumer": needs test.provider v2, which plugin "@test/provider" serves on its old code', + ); + expect(shown.label.value).toBe("v1 1"); + await host.close(context); + plugins.close(); + }); +}); + +describe("rollout", () => { + /** A step recording its name into `order`, answering `report`, or rejecting with `error`. */ + const step = (name: string, order: string[], outcome: { report?: ReloadReport; error?: Error }): RolloutStep => ({ + name, + kinds: ["worker"], + reload: async () => { + order.push(name); + if (outcome.error !== undefined) throw outcome.error; + return outcome.report!; + }, + }); + const report = (failed: readonly string[]): ReloadReport => ({ + plugins: { added: [], changed: [], removed: [] }, + failed: failed.map((plugin) => ({ plugin, error: new Error("x") })), + objects: [], + }); + + it("runs the steps in order, clears the marks of plugins applied on every step, keeps a plugin failed on any", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// a\n"); + await writeFile(join(root, "b", "worker.ts"), "// b\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a", "@test/b"] }), { + timeout: 5_000, + }); + const built = await plugins.build(["@test/a", "@test/b"], context); + const order: string[] = []; + const result = await plugins.rollout( + [step("server", order, { report: report([]) }), step("worker", order, { report: report(["@test/b"]) })], + built, + context, + ); + expect([order, result.stopped, result.steps.map(({ name }) => name)]).toEqual([ + ["server", "worker"], + undefined, + ["server", "worker"], + ]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/b"] }); + plugins.close(); + }); + + it("stops at a step that rejects, clearing no mark; a change during the build keeps its mark", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + await plugins.build([], context); + await writeFile(join(root, "a", "worker.ts"), "// 1\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + const order: string[] = []; + const stopped = await plugins.rollout( + [step("server", order, { error: new Error("closed") }), step("worker", order, { report: report([]) })], + built, + context, + ); + expect([order, stopped.stopped?.step, (stopped.stopped?.error as Error).message]).toEqual([ + ["server"], + "server", + "closed", + ]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + // Saved again after the build: applied, the mark stays (the build did not include the new content). + const later = await plugins.build(["@test/a"], context); + await writeFile(join(root, "a", "worker.ts"), "// 2\n"); + await vi.waitFor(async () => { + await plugins.rollout([step("server", [], { report: report([]) })], later, context); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + }); + plugins.close(); + }); +}); + +describe("installs through the component", () => { + it("clears the plugin's install mark and dirties it", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const reported: unknown[] = []; + const plugins = createPlugins( + { + roots: [root], + kinds: KINDS, + hostPackages: HOST_PACKAGES, + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: (error) => reported.push(error), + }, + context, + ); + await plugins.build([], context); + await writeFile( + join(root, "a", "package.json"), + JSON.stringify({ name: "@test/a", type: "module", dependencies: {} }), + ); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ install: ["@test/a"] }), { timeout: 5_000 }); + await plugins.installDependencies("@test/a", context); + expect(plugins.state.value).toMatchObject({ install: [], dirty: ["@test/a"] }); + expect(reported).toEqual([]); + plugins.close(); + }); +}); + +describe("GC", () => { + it("collects old facets over 200 reloads; the heap stays flat after warm-up", async () => { + const root = await tempRoot(); + const made: WeakRef[] = []; + const probe = { made: (closure: object) => void made.push(new WeakRef(closure)) }; + const source = (label: string) => + [ + 'import { constant, defineFacet, local, str } from "@earendil-works/chord/next";', + 'import { made } from "@test/probe";', + 'const Item = local("test.gc", { version: 1, params: {}, members: { label: str() } });', + // A closure per evaluation, which only this evaluation's code holds. + `const state = { label: ${JSON.stringify(label)}, big: new Array(10_000).fill(${JSON.stringify(label)}) };`, + "made(state);", + "export default defineFacet((host) => host.serve(Item, () => ({ label: constant(state.label) })));", + ].join("\n"); + await writePlugin(root, "gc", "@test/gc", { "worker.ts": source("a") }); + const plugins = createPlugins( + { roots: [root], kinds: KINDS, hostPackages: { ...HOST_PACKAGES, "@test/probe": probe }, onError: () => {} }, + context, + ); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + const first = await plugins.build(["@test/gc"], context); + await writeFiles(join(root, "gc"), { "worker.ts": source("b") }); + const second = await plugins.build(["@test/gc"], context); + const reload = async (count: number) => { + for (let i = 0; i < count; i++) await plugins.reload(host, i % 2 === 0 ? second : first, context); + }; + await reload(20); + const warm = await settledHeap(); + await reload(200); + const after = await settledHeap(); + expect(after - warm).toBeLessThan(2 * 1024 * 1024); + await expectCollected(...made.slice(0, made.length - 2)); + await host.close(context); + plugins.close(); + }); +}); + +describe("end to end: a server and a worker", () => { + it("loads plugin directories, rolls an edit out (server, then worker), and keeps a broken edit old on both", async () => { + type P = { readonly kind: "server" } | { readonly kind: "worker" }; + const root = await tempRoot(["contracts", "notes"]); + const { plugins } = component(root); + const [up, down] = memoryLinkPair(); + const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); + worker.connect(up, { principal: { kind: "server" } }, context); + const server = createHost

({ principal: { kind: "server" }, onError: () => {} }); + server.route( + Worker, + { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, + context, + ); + await plugins.load(worker, { kind: "worker", core: [] }, context); + await plugins.load(server, { kind: "server", core: [] }, context); + const notes = await server.get(Notes, {}, context); + const index = await server.get(Index, {}, context); + expect([notes.label.value, index.label.value]).toEqual(["v1", "v1"]); + await notes.add(context); + const frames = changesOf(await notes.label.watch(context)); + const steps = [plugins.step("server", server as Host

), plugins.step("worker", worker as Host

)]; + + await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); + const rolled = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); + expect(rolled.steps.map(({ name, report }) => [name, report.plugins.changed, report.objects])).toEqual([ + ["server", ["@test/notes"], [{ object: "test.index", kind: "swapped" }]], + ["worker", ["@test/notes"], [{ object: "test.notes", kind: "swapped" }]], + ]); + await transported(); + await delivered(); + expect(valuesOf(frames)).toEqual(["v2"]); + expect([notes.label.value, notes.count.value, index.label.value]).toEqual(["v2", 1, "v2"]); + expect(plugins.state.value).toMatchObject({ dirty: [] }); + + await writeFile(join(root, "notes", "label.ts"), "export const label = ;\n"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }), { timeout: 5_000 }); + const broken = await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context); + expect(broken.steps.map(({ name, report }) => [name, failedNames(report)])).toEqual([ + ["server", ["@test/notes"]], + ["worker", ["@test/notes"]], + ]); + await transported(); + expect([notes.label.value, index.label.value, notes.state.value]).toEqual(["v2", "v2", { kind: "live" }]); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/notes"] }); + await server.close(context); + await worker.close(context); + plugins.close(); + }); +}); + +describe("fix round 1", () => { + it("serializes reloads of one host: two started together keep both changes (Astra, Sol, Opus S1)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + await itemPlugin(root, "a", "a2"); + const builtA = await plugins.build(["@test/a"], context); + await itemPlugin(root, "b", "b2"); + const builtB = await plugins.build(["@test/b"], context); + const reports = await Promise.all([plugins.reload(host, builtA, context), plugins.reload(host, builtB, context)]); + expect(reports.map(({ plugins: { changed } }) => changed)).toEqual([["@test/a"], ["@test/b"]]); + const label = async (dir: string) => + (await host.get(local(`test.${dir}`, { version: 1, params: {}, members: { label: str() } }), {}, context)) + .label.value; + expect([await label("a"), await label("b")]).toEqual(["a2", "b2"]); + await host.close(context); + plugins.close(); + }); + + it("applies the removals of the build it was given, never a later discovery's (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + await itemPlugin(root, "a", "a2"); + const first = await plugins.build(["@test/a"], context); + await rm(join(root, "b"), { recursive: true }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const later = await plugins.build([], context); + expect([first.removed, later.removed]).toEqual([[], ["@test/b"]]); + const report = await plugins.reload(host, first, context); + expect(report.plugins).toEqual({ added: [], changed: ["@test/a"], removed: [] }); + expect((await plugins.reload(host, later, context)).plugins.removed).toEqual(["@test/b"]); + await host.close(context); + plugins.close(); + }); + + it("adopts what a reload committed when its wait aborts after the commit: a later reload keeps it (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + await itemPlugin(root, "b", "b1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + const Item = local("test.a", { version: 1, params: {}, members: { label: str() } }); + const item = await host.get(Item, {}, context); + await itemPlugin(root, "a", "a2"); + const first = await plugins.build(["@test/a"], context); + // The abort lands right after the reload's commit (its publication batch), while it waits for its report. + const { batch } = host as ChordHost; + const commit = batch.commit.bind(batch); + const controller = new AbortController(); + const aborting = withAbortSignal(controller.signal, context); + const hook = vi.spyOn(batch, "commit").mockImplementation((adopt, ctx) => { + commit(adopt, ctx); + if (ctx === aborting) controller.abort(); + }); + expect(await rejection(plugins.reload(host, first, aborting))).toMatchObject({ + code: "aborted", + details: { committed: { added: [], changed: ["@test/a"], removed: [], failed: [] } }, + }); + hook.mockRestore(); + expect(item.label.value).toBe("a2"); + await itemPlugin(root, "b", "b2"); + await plugins.reload(host, await plugins.build(["@test/b"], context), context); + expect(item.label.value).toBe("a2"); + await host.close(context); + plugins.close(); + }); + + it("leaves a plugin out of a host whose kind its rebuilt facets no longer have (Opus recheck R1)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + // The plugin keeps a server facet only. + await rm(join(root, "a", "worker.ts")); + await writeFiles(join(root, "a"), { + "server.ts": + 'import { defineFacet } from "@earendil-works/chord/next";\nexport default defineFacet(() => {});', + }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ set: true }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + expect(built.removed).toEqual([]); + expect((await plugins.reload(host, built, context)).plugins).toEqual({ + added: [], + changed: [], + removed: ["@test/a"], + }); + await host.close(context); + plugins.close(); + }); + + it("reloads a structured clone of a build as the build itself: it is plain data (Astra)", async () => { + const root = await tempRoot(); + await itemPlugin(root, "a", "a1"); + const { plugins } = component(root); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + const item = await host.get(local("test.a", { version: 1, params: {}, members: { label: str() } }), {}, context); + await itemPlugin(root, "a", "a2"); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + const built = await plugins.build(["@test/a"], context); + expect((await plugins.reload(host, structuredClone(built), context)).plugins.changed).toEqual(["@test/a"]); + expect(item.label.value).toBe("a2"); + // The marks clear only through the build object itself (a copy carries no snapshot). + const result = await plugins.rollout([plugins.step("worker", host)], structuredClone(built), context); + expect(result.stopped).toBeUndefined(); + expect(plugins.state.value).toMatchObject({ dirty: ["@test/a"] }); + await host.close(context); + plugins.close(); + }); + + it("reads a changed package.json again at the next build: a new dependency is accepted (Astra, Sol)", async () => { + const root = await tempRoot(); + const tools = await tempRoot(); + await writeFiles(tools, { "npm.mjs": "process.exit(0);" }); + const dir = await writePlugin(root, "a", "@test/a", { + "worker.ts": 'import { label } from "./helper.ts";\nexport default { setup: () => {}, label };', + "helper.ts": 'export const label = "old";', + }); + const plugins = createPlugins( + { + roots: [root], + kinds: KINDS, + hostPackages: HOST_PACKAGES, + install: { npmCommand: [process.execPath, join(tools, "npm.mjs")] }, + onError: () => {}, + }, + context, + ); + expect((await plugins.build([], context)).facets.map(({ kind }) => kind)).toEqual(["script"]); + await writeFiles(dir, { + "package.json": JSON.stringify({ name: "@test/a", type: "module", dependencies: { "new-dep": "1.0.0" } }), + }); + await vi.waitFor(() => expect(plugins.state.value).toMatchObject({ install: ["@test/a"] }), { timeout: 5_000 }); + await writePackage(dir, "new-dep", { "index.js": 'exports.label = "new";' }); + await writeFiles(dir, { "helper.ts": 'export { label } from "new-dep";' }); + await plugins.installDependencies("@test/a", context); + expect(plugins.state.value).toMatchObject({ set: false }); + const built = await plugins.build(["@test/a"], context); + expect(built.facets).toEqual([expect.objectContaining({ kind: "script", plugin: "@test/a" })]); + expect(plugins.discovery.plugins[0]!.dependencies).toEqual(["new-dep"]); + plugins.close(); + }); +}); + +describe("the reload queues and garbage collection", () => { + it("lets a rejected new facet go once its report is dropped, the kept old one live (Sol)", async () => { + const root = await tempRoot(); + const made: WeakRef[] = []; + const probe = { made: (value: object) => made.push(new WeakRef(value)) }; + const source = (bad: boolean) => + [ + 'import { constant, defineFacet, int, local } from "@earendil-works/chord/next";', + 'import { made } from "@test/probe";', + `const state = { bad: ${bad}, big: new Array(10000).fill("payload") };`, + "made(state);", + 'const Type = local("test.failed-gc", { version: 1, params: {}, members: { n: int() } });', + "export default defineFacet((h) => {", + '\tif (state.bad) throw new Error("bad setup");', + "\th.serve(Type, () => ({ n: constant(state.big.length) }));", + "});", + ].join("\n"); + const dir = await writePlugin(root, "failed", "@test/failed", { "worker.ts": source(false) }); + const plugins = createPlugins( + { roots: [root], kinds: KINDS, hostPackages: { ...HOST_PACKAGES, "@test/probe": probe }, onError: () => {} }, + context, + ); + const host = createHost({ onError: () => {} }); + await plugins.load(host, { kind: "worker", core: [] }, context); + await writeFiles(dir, { "worker.ts": source(true) }); + const built = await plugins.build(["@test/failed"], context); + // The report (its failure's error holds the new facet's closure) is dropped here; the queues must not keep it. + expect((await plugins.reload(host, built, context)).failed.map(({ plugin }) => plugin)).toEqual(["@test/failed"]); + await expectCollected(made[1]!); + expect(made[0]!.deref()).toBeDefined(); + await host.close(context); + plugins.close(); + }); +}); diff --git a/packages/chord/test/next-node/plugins/stack.test.ts b/packages/chord/test/next-node/plugins/stack.test.ts new file mode 100644 index 000000000..92ec48fc6 --- /dev/null +++ b/packages/chord/test/next-node/plugins/stack.test.ts @@ -0,0 +1,123 @@ +// Tests of stack traces (slice 7c §3, decision Q1 (b)): `mapStack` names plugin files and lines through each +// evaluated script's source map; nothing global; a script's map goes with its facet. + +import { describe, expect, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { + type BuiltScript, + buildFacet, + discoverPlugins, + evaluateFacet, + mapStack, + type Plugin, +} from "../../../src/next-node/plugins/index.ts"; +import { expectCollected, settledHeap } from "../../next/support.ts"; +import { HOST_PACKAGES, KINDS, tempRoot, writePlugin } from "./support.ts"; + +/** A plugin `notes` whose worker facet's `fail` throws from `helper.ts` line 3. */ +async function failing(): Promise<{ script: BuiltScript; plugin: Plugin }> { + const root = await tempRoot(); + await writePlugin(root, "notes", "@test/notes", { + "worker.ts": 'import { helper } from "./helper.ts";\nexport default { setup: () => {}, fail: () => helper() };', + "helper.ts": + '// A helper.\nexport function helper(): never {\n\tthrow new Error("helper failed at " + Date.now());\n}', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet( + plugin, + "worker", + { hostPackages: Object.keys(HOST_PACKAGES), platform: "node" }, + context, + ); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + return { script: built, plugin }; +} + +/** The error `facet`'s `fail` throws. */ +function thrownBy(facet: object): unknown { + try { + (facet as { readonly fail: () => void }).fail(); + } catch (error) { + return error; + } + throw new Error("fail did not throw"); +} + +describe("mapStack", () => { + it("maps a plugin script's positions to the plugin's files and lines; other lines stay; nothing global", async () => { + const { script, plugin } = await failing(); + const prepare = Error.prepareStackTrace; + const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + const error = thrownBy(facet); + const mapped = mapStack(error); + expect(mapped).toMatch(/at \S+ \(notes\/helper\.ts:3:8\)/); + expect(mapped).toMatch(/notes\/worker\.ts:2:\d+/); + expect(mapped).not.toContain("/chord-plugins/"); + // The raw stack is untouched: no `Error.prepareStackTrace` of the component's. + expect((error as Error).stack).toContain("/chord-plugins/@test/notes/worker.ts?"); + expect(Error.prepareStackTrace).toBe(prepare); + expect(mapStack("not an error")).toBe("not an error"); + }); + + it("lets a script's source map go with its facet", async () => { + const { script, plugin } = await failing(); + const error = await (async () => { + const facet = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + return { error: thrownBy(facet), ref: new WeakRef(facet) }; + })(); + expect(mapStack(error.error)).toContain("notes/helper.ts:3:8"); + await expectCollected(error.ref); + // The finalization runs after the collection. + for (let round = 0; round < 10 && mapStack(error.error).includes("notes/helper.ts"); round++) await settledHeap(); + expect(mapStack(error.error)).toContain("/chord-plugins/@test/notes/worker.ts?"); + }); +}); + +describe("mapStack, fix round 1", () => { + it("maps an import-time throw (the map is registered before the script runs) (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "evaluation", "@test/evaluation", { + "worker.ts": + '// An import-time throw.\n\nthrow new Error("at import");\nexport default { setup: () => {} };\n', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet(plugin, "worker", { hostPackages: [], platform: "node" }, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + let failure: unknown; + try { + evaluateFacet(built, { hostPackages: {}, plugin }); + } catch (error) { + failure = error; + } + expect(mapStack(failure)).toContain("evaluation/worker.ts:3:"); + }); + + it("maps a throw while its default export is read (a getter) (Sol)", async () => { + const root = await tempRoot(); + await writePlugin(root, "getter", "@test/getter", { + "worker.ts": '// A getter failure.\nexport default {\n\tget setup() { throw new Error("getter boom"); } };\n', + }); + const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); + const plugin = plugins[0]!; + const built = await buildFacet(plugin, "worker", { hostPackages: [], platform: "node" }, context); + if (built.kind === "failed") throw new Error(built.errors.join("\n")); + let failure: unknown; + try { + evaluateFacet(built, { hostPackages: {}, plugin }); + } catch (error) { + failure = error; + } + expect(mapStack(failure)).toContain("getter/worker.ts:3:"); + }); + + it("keeps a script's map while any evaluation of it lives: two hosts share it (Sol, Opus N3)", async () => { + const { script, plugin } = await failing(); + const first = evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin }); + const second = (() => new WeakRef(evaluateFacet(script, { hostPackages: HOST_PACKAGES, plugin })))(); + await expectCollected(second); + for (let round = 0; round < 5; round++) await settledHeap(); + expect(mapStack(thrownBy(first))).toContain("notes/helper.ts:3:8"); + }); +}); diff --git a/packages/chord/test/next-node/plugins/support.ts b/packages/chord/test/next-node/plugins/support.ts new file mode 100644 index 000000000..86e444156 --- /dev/null +++ b/packages/chord/test/next-node/plugins/support.ts @@ -0,0 +1,66 @@ +// Test support for the plugins component (slice 7c): fixture plugins copied into temporary roots, plugins and npm +// packages written there, and the host packages the fixtures import. + +import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterEach } from "vitest"; +import * as chordNext from "../../../src/next/index.ts"; +import type { HostPackages } from "../../../src/next-node/plugins/index.ts"; + +/** The fixture plugins' directory. */ +export const FIXTURES = fileURLToPath(new URL("../fixtures/plugins/", import.meta.url)); + +/** The host packages the fixtures import: this process's Chord. */ +export const HOST_PACKAGES: HostPackages = { "@earendil-works/chord/next": chordNext }; + +/** The facet kinds of the fixtures' app. */ +export const KINDS = ["server", "worker", "web"] as const; + +const made: string[] = []; + +afterEach(async () => { + for (const dir of made.splice(0)) await rm(dir, { recursive: true, force: true }); +}); + +/** A temporary directory removed after the test; `fixtures` (names of fixture plugins) copied into it. */ +export async function tempRoot(fixtures: readonly string[] = []): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-plugins-")); + made.push(root); + for (const name of fixtures) await cp(join(FIXTURES, name), join(root, name), { recursive: true }); + return root; +} + +/** Writes `files` (relative path to text) into `dir`, creating directories. */ +export async function writeFiles(dir: string, files: Readonly>): Promise { + for (const [path, text] of Object.entries(files)) { + await mkdir(dirname(join(dir, path)), { recursive: true }); + await writeFile(join(dir, path), text); + } +} + +/** Writes a plugin named `name` into `root/dir`: its `package.json` (with `manifest`'s fields) and `files`. */ +export async function writePlugin( + root: string, + dir: string, + name: string, + files: Readonly>, + manifest: Readonly> = {}, +): Promise { + const path = join(root, dir); + await writeFiles(path, { "package.json": JSON.stringify({ name, type: "module", ...manifest }), ...files }); + return path; +} + +/** Writes an npm package `name` into `dir`'s `node_modules`: its `package.json` (CommonJS unless `type`) and files. */ +export async function writePackage( + dir: string, + name: string, + files: Readonly>, + manifest: Readonly> = {}, +): Promise { + const main = Object.keys(files)[0]!; + const json = JSON.stringify({ name, version: "1.0.0", main, ...manifest }); + await writeFiles(join(dir, "node_modules", name), { "package.json": json, ...files }); +} diff --git a/packages/chord/test/next-node/plugins/watcher.test.ts b/packages/chord/test/next-node/plugins/watcher.test.ts new file mode 100644 index 000000000..8e0a912e1 --- /dev/null +++ b/packages/chord/test/next-node/plugins/watcher.test.ts @@ -0,0 +1,202 @@ +// Tests of the watcher (slice 7c §5): dirty marks from the files facets used and from plugin directories, `install`, +// `set`, `base`, a change during a build keeping its mark, `node_modules` and `.git` ignored, real file events, and the +// marks as a JSON watchable. + +import { readdirSync, readFileSync } from "node:fs"; +import { realpath, rm, symlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { Batch } from "../../../src/next/state/internal.ts"; +import { type BuiltFacet, discoverPlugins } from "../../../src/next-node/plugins/index.ts"; +import { PluginWatcher } from "../../../src/next-node/plugins/watcher.ts"; +import { changesOf, delivered } from "../../next/support.ts"; +import { KINDS, tempRoot, writeFiles, writePlugin } from "./support.ts"; + +/** A root with plugins `a` (worker), `b` (server) and the contracts fixture, a base directory, and a watcher of both. */ +async function watched() { + const root = await realpath(await tempRoot(["contracts"])); + const base = await realpath(await tempRoot()); + await writePlugin(root, "a", "@test/a", { "worker.ts": "", "own.ts": "" }); + await writePlugin(root, "b", "@test/b", { "server.ts": "" }); + const failures: unknown[] = []; + const watcher = new PluginWatcher( + { roots: [root], base: [base], kinds: KINDS, failed: (error) => failures.push(error) }, + new Batch(() => {}, undefined), + ); + const discovery = await discoverPlugins([root], { kinds: KINDS }, context); + watcher.know(discovery, watcher.discoverySnapshot()); + const contract = join(root, "contracts", "contract.ts"); + const used = (plugin: string, facet: string, files: readonly string[]): BuiltFacet => ({ + kind: "script", + plugin, + facet, + source: "", + digest: "", + inputs: files.map((file) => ({ file, hash: "" })), + copied: [contract], + assets: [], + }); + watcher.used([ + used("@test/a", "worker", [join(root, "a", "worker.ts"), contract]), + used("@test/b", "server", [join(root, "b", "server.ts"), contract]), + ]); + return { root, base, watcher, failures }; +} + +describe("the watcher's marks", () => { + it("dirties every plugin whose facets used a changed file, and the plugin whose directory contains it", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "contracts", "contract.ts")); + expect(watcher.value).toEqual({ + dirty: ["@test/a", "@test/b", "@test/contracts"], + install: [], + set: false, + base: false, + }); + watcher.close(); + }); + + it("dirties the plugin of a file no facet read yet; asks for an install for a package.json", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "new.ts")); + watcher.mark(join(root, "b", "package.json")); + expect(watcher.value).toEqual({ dirty: ["@test/a"], install: ["@test/b"], set: false, base: false }); + // Installed: the mark clears, and the plugin is dirty (its packages changed). + watcher.installed("@test/b"); + expect(watcher.value).toEqual({ dirty: ["@test/a", "@test/b"], install: [], set: false, base: false }); + watcher.close(); + }); + + it("asks for discovery for a directory under a root and a facet file at a plugin's top; base for base code", async () => { + const { root, base, watcher } = await watched(); + watcher.mark(join(root, "c")); + expect(watcher.value).toMatchObject({ set: true, dirty: [] }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: false }); + watcher.mark(join(root, "b", "web.tsx")); + expect(watcher.value).toMatchObject({ set: true, dirty: ["@test/b"] }); + watcher.mark(join(base, "core.ts")); + expect(watcher.value).toMatchObject({ base: true }); + watcher.close(); + }); + + it("ignores node_modules and .git", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "node_modules", "x", "index.js")); + watcher.mark(join(root, "a", ".git", "HEAD")); + expect(watcher.value).toEqual({ dirty: [], install: [], set: false, base: false }); + watcher.close(); + }); + + it("keeps the mark of a change during a build: only an unchanged mark clears when applied", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "a", "own.ts")); + const snapshot = watcher.snapshot(["@test/a"]); + // Saved again while the build runs. + watcher.mark(join(root, "a", "own.ts")); + watcher.applied(["@test/a"], snapshot); + expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }); + watcher.applied(["@test/a"], watcher.snapshot(["@test/a"])); + expect(watcher.value).toMatchObject({ dirty: [] }); + // A discovery that started before a change of the set keeps `set`. + const seen = watcher.discoverySnapshot(); + watcher.mark(join(root, "d")); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ set: true }); + watcher.close(); + }); + + it("marks from real file events, publishing the marks to a watch", async () => { + const { root, watcher } = await watched(); + const changes = changesOf(await watcher.watch(context)); + await writeFile(join(root, "a", "node_modules-free.ts"), "export {};"); + await writeFiles(join(root, "b", "node_modules", "pkg"), { "index.js": "" }); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + await delivered(); + expect(changes.at(-1)).toMatchObject({ kind: "value", value: { dirty: ["@test/a"] } }); + watcher.close(); + }); +}); + +describe("the watcher, fix round 1", () => { + it("never lets an older build's applied snapshot clear a newer edit's mark (Astra)", async () => { + const { root, watcher } = await watched(); + const file = join(root, "a", "own.ts"); + watcher.mark(file); + const first = watcher.snapshot(["@test/a"]); + watcher.mark(file); + const second = watcher.snapshot(["@test/a"]); + watcher.applied(["@test/a"], second); + expect(watcher.value).toMatchObject({ dirty: [] }); + watcher.mark(file); + watcher.applied(["@test/a"], first); + expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }); + watcher.close(); + }); + + it("drops the install mark of a plugin the next discovery no longer has (example 38)", async () => { + const { root, watcher } = await watched(); + watcher.mark(join(root, "b", "package.json")); + expect(watcher.value).toMatchObject({ install: ["@test/b"] }); + await rm(join(root, "b"), { recursive: true }); + const seen = watcher.discoverySnapshot(); + watcher.know(await discoverPlugins([root], { kinds: KINDS }, context), seen); + expect(watcher.value).toMatchObject({ install: [] }); + watcher.close(); + }); + + it.skipIf(process.platform !== "linux")( + "puts no watch on node_modules or .git (Opus N2): 200 packages add no inotify watch", + async () => { + const root = await realpath(await tempRoot()); + const dir = await writePlugin(root, "a", "@test/a", { "worker.ts": "" }); + for (let i = 0; i < 200; i++) + await writeFiles(join(dir, "node_modules", `pkg${i}`, "lib"), { "index.js": "" }); + await writeFiles(join(dir, ".git", "objects"), { x: "" }); + const before = inotifyWatches(); + const watcher = new PluginWatcher( + { roots: [root], base: [], kinds: KINDS, failed: () => {} }, + new Batch(() => {}, undefined), + ); + // The root and the plugin's directory. + expect(inotifyWatches() - before).toBe(2); + watcher.close(); + }, + ); + + it("watches a directory that appears later, and marks the files in it", async () => { + const { root, watcher } = await watched(); + await writeFiles(join(root, "a", "lib"), { "first.ts": "" }); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.applied(["@test/a"], watcher.snapshot(["@test/a"])); + await writeFile(join(root, "a", "lib", "second.ts"), "export {};"); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.close(); + }); +}); + +describe("the watcher and links", () => { + it("survives a link loop created under a root (its stat never escapes the callback) and still marks it (Sol)", async () => { + const { root, watcher } = await watched(); + await symlink("loop", join(root, "a", "loop")); + await vi.waitFor(() => expect(watcher.value).toMatchObject({ dirty: ["@test/a"] }), { timeout: 5_000 }); + watcher.close(); + }); +}); + +/** The inotify watches this process holds (Linux): one line per watch in its inotify descriptors' fdinfo. */ +function inotifyWatches(): number { + let count = 0; + for (const fd of readdirSync("/proc/self/fdinfo")) { + try { + count += readFileSync(`/proc/self/fdinfo/${fd}`, "utf8") + .split("\n") + .filter((line) => line.startsWith("inotify wd:")).length; + } catch { + // Closed meanwhile. + } + } + return count; +} diff --git a/packages/chord/test/next/bench/plugins.ts b/packages/chord/test/next/bench/plugins.ts new file mode 100644 index 000000000..3cd231ebb --- /dev/null +++ b/packages/chord/test/next/bench/plugins.ts @@ -0,0 +1,146 @@ +// Benchmark of the plugins component (slice 7c §7): a cold build of a 20-plugin set (one worker facet each, its own +// files, a copied contract), a rebuild of one facet after an edit, the evaluation of a facet (`vm.Script`), and the heap +// after 100 reloads of one plugin on a host (old facets collected, V8's compilation cache off). Medians over runs. +// Run by hand from packages/chord (not in CI), pinned for comparisons: +// taskset -c 6 node --conditions=source --experimental-strip-types --expose-gc test/next/bench/plugins.ts +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { performance } from "node:perf_hooks"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; +import { createHost } from "../../../src/next/host/index.ts"; +import * as chordNext from "../../../src/next/index.ts"; +import { + type BuiltScript, + buildFacet, + createPlugins, + discoverPlugins, + evaluateFacet, +} from "../../../src/next-node/plugins/index.ts"; + +const PLUGINS = 20; +const RUNS = 7; +const RELOADS = 100; + +const gc = (globalThis as { gc?: () => void }).gc!; +const HOST_PACKAGES = { "@earendil-works/chord/next": chordNext }; +const KINDS = ["server", "worker"]; + +/** A plugin's worker facet: a store over a document, a derive, a method, a helper module and the shared contract. */ +function worker(index: number, label: string): string { + return [ + 'import { constant, defineFacet, defineState, int, local, method, obj, str } from "@earendil-works/chord/next";', + 'import { Shared } from "../contracts/contract.ts";', + 'import { format } from "./format.ts";', + `const Store = local("bench.store${index}", { version: 1, params: {}, members: { label: str(), n: int(), bump: method(undefined, int()) } });`, + 'const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) });', + "export default defineFacet((host) => host.serve(Store, (_, self) => {", + " const doc = self.state(Count);", + ` return { label: constant(format(${JSON.stringify(label)}, Shared.name)), n: self.derive([doc], ([d]) => d.n),`, + " bump: async (ctx) => { await doc.change((d) => { d.n += 1; }, ctx); return doc.value.n; } };", + "}));", + ].join("\n"); +} + +/** Writes the 20-plugin set (and the contracts plugin) under a new root. */ +async function pluginSet(): Promise { + const root = await mkdtemp(join(tmpdir(), "chord-bench-plugins-")); + const write = async (dir: string, files: Record) => { + await mkdir(join(root, dir), { recursive: true }); + for (const [file, text] of Object.entries(files)) await writeFile(join(root, dir, file), text); + }; + await write("contracts", { + "package.json": JSON.stringify({ name: "@bench/contracts", type: "module" }), + "contract.ts": + 'import { local, str } from "@earendil-works/chord/next";\nexport const Shared = local("bench.shared", { version: 1, params: {}, members: { name: str() } });', + }); + for (let index = 0; index < PLUGINS; index++) { + await write(`p${index}`, { + "package.json": JSON.stringify({ name: `@bench/p${index}`, type: "module" }), + "worker.ts": worker(index, "v1"), + "format.ts": + 'export function format(label: string, name: string): string {\n return label + " (" + name + ")";\n}', + }); + } + return root; +} + +/** Median of `values`. */ +function median(values: number[]): number { + return values.sort((a, b) => a - b)[Math.floor(values.length / 2)]!; +} + +const root = await pluginSet(); +const { plugins } = await discoverPlugins([root], { kinds: KINDS }, context); +const options = { hostPackages: Object.keys(HOST_PACKAGES), platform: "node", plugins } as const; +const facets = plugins.filter((plugin) => plugin.facets.worker !== undefined); + +// A cold build of every worker facet (each its own esbuild build, all at once as the component runs them). +const cold: number[] = []; +let scripts: BuiltScript[] = []; +for (let run = 0; run < RUNS; run++) { + const started = performance.now(); + const built = await Promise.all(facets.map((plugin) => buildFacet(plugin, "worker", options, context))); + cold.push(performance.now() - started); + scripts = built.filter((each): each is BuiltScript => each.kind === "script"); +} +const bytes = scripts.reduce((sum, each) => sum + each.source.length, 0); +console.log( + `cold build of ${facets.length} worker facets: ${median(cold).toFixed(1)} ms (median of ${RUNS}), ${(bytes / 1024).toFixed(1)} KiB of scripts`, +); + +// A rebuild of one facet after an edit. +const rebuild: number[] = []; +for (let run = 0; run < RUNS; run++) { + await writeFile(join(root, "p0", "worker.ts"), worker(0, `edit ${run}`)); + const started = performance.now(); + await buildFacet(facets[0]!, "worker", options, context); + rebuild.push(performance.now() - started); +} +console.log(`rebuild of one facet: ${median(rebuild).toFixed(1)} ms (median of ${RUNS})`); + +// The evaluation of a facet: compile and run its script. +const evaluations: number[] = []; +for (let run = 0; run < RUNS; run++) { + const started = performance.now(); + for (let index = 0; index < scripts.length; index++) { + evaluateFacet(scripts[index]!, { hostPackages: HOST_PACKAGES, plugin: facets[index]! }); + } + evaluations.push((performance.now() - started) / scripts.length); +} +console.log(`evaluation per facet: ${median(evaluations).toFixed(2)} ms (median of ${RUNS})`); + +// The heap after 100 reloads of one plugin on a host, against after a warm-up (two builds alternating). +const component = createPlugins( + { roots: [root], kinds: KINDS, hostPackages: HOST_PACKAGES, onError: () => {} }, + context, +); +const host = createHost({ onError: () => {} }); +await component.load(host, { kind: "worker", core: [] }, context); +await writeFile(join(root, "p0", "worker.ts"), worker(0, "a")); +const first = await component.build(["@bench/p0"], context); +await writeFile(join(root, "p0", "worker.ts"), worker(0, "b")); +const second = await component.build(["@bench/p0"], context); +const settled = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setImmediate(resolve)); + gc(); + } + return process.memoryUsage().heapUsed; +}; +for (let reload = 0; reload < 10; reload++) await component.reload(host, reload % 2 === 0 ? second : first, context); +const warm = await settled(); +const reloadTimes: number[] = []; +for (let reload = 0; reload < RELOADS; reload++) { + const started = performance.now(); + await component.reload(host, reload % 2 === 0 ? second : first, context); + reloadTimes.push(performance.now() - started); +} +const after = await settled(); +console.log( + `reload of one plugin (evaluation and host reload): ${median(reloadTimes).toFixed(2)} ms (median of ${RELOADS})`, +); +console.log(`heap after ${RELOADS} reloads: ${((after - warm) / 1024).toFixed(0)} KiB against after the warm-up`); +await host.close(context); +component.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/35-plugins/main.ts b/packages/chord/test/next/examples/35-plugins/main.ts new file mode 100644 index 000000000..bdfaa89d9 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/main.ts @@ -0,0 +1,99 @@ +// Plugins (slice 7c): plugin directories turned into facets, loaded on their hosts, watched, rebuilt and rolled out. +// A plugin is a directory with a `package.json` (its name) and one file per facet kind (`server.ts`, `worker.ts`); +// `createPlugins` discovers them, builds each facet into a script with esbuild (host packages external) and evaluates +// it on Node. Here (`plugins/`): a contracts-only plugin, `@example/notes` with a server and a worker facet showing one +// label, and `@example/clock`, which the example never edits. A server and a worker load them; the notes' label is +// edited on disk, the watcher marks the plugin, the app builds it and rolls it out (server, then worker), a holder sees +// one frame, and the clock plugin is untouched (not in the reports, its object the same). +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/35-plugins/main.ts +import { cp, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost, type Host } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { memoryLinkPair } from "../../../../src/next/link/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import { Clock, Index, Notes, Worker } from "./plugins/contracts/contract.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example edits files: it works on a copy of `plugins/`, removed at the end. +const root = await mkdtemp(join(tmpdir(), "chord-example-35-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +// 1. The component: the roots, the facet kinds, and the host packages every script gets (this process's Chord: a +// plugin's `import { defineFacet } from "@earendil-works/chord/next"` is the app's own instance). +const plugins = createPlugins( + { + roots: [root], + kinds: ["server", "worker"], + hostPackages: { "@earendil-works/chord/next": chordNext }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. A worker and a server joined by a memory link; each loads the plugins' facets of its kind (no core facets here). +type P = { readonly kind: "server" } | { readonly kind: "worker" }; +const [up, down] = memoryLinkPair(); +const worker = createHost

({ address: Worker(), principal: { kind: "worker" }, onError: () => {} }); +worker.connect(up, { principal: { kind: "server" } }, context); +const server = createHost

({ principal: { kind: "server" }, onError: () => {} }); +server.route( + Worker, + { authorize: () => true, resolve: async () => ({ link: down, principal: { kind: "worker" } }) }, + context, +); +const workerLoad = await plugins.load(worker, { kind: "worker", core: [] }, context); +const serverLoad = await plugins.load(server, { kind: "server", core: [] }, context); +console.log( + "1. discovered:", + plugins.discovery.plugins.map(({ name, facets }) => `${name} [${Object.keys(facets)}]`).join(", "), +); +console.log(" load failures: worker", workerLoad.failed.length, "| server", serverLoad.failed.length); +const notes = await server.get(Notes, {}, context); +const index = await server.get(Index, {}, context); +const clock = await server.get(Clock, {}, context); +await notes.add(context); +const evaluatedAt = clock.evaluatedAt.value; +console.log(" notes:", notes.label.value, "count", notes.count.value, "| index:", index.label.value); +const watch = await notes.label.watch(context); +watch.start(async (call) => { + if (call.kind === "value") console.log(" a frame on the server's reference:", JSON.stringify(call.value)); +}); + +// 3. An edit on disk (the app's editor, a git pull): the watcher marks the plugin dirty; nothing is applied yet. +await writeFile(join(root, "notes", "label.ts"), 'export const label = "v2";\n'); +await until(() => plugins.state.value.dirty.includes("@example/notes")); +console.log("2. marks after the edit:", JSON.stringify(plugins.state.value)); + +// 4. The app builds the dirty plugins and rolls the build out: the server first, then the worker. Each step reloads its +// host with the facets of its kind; the clock plugin was not built, so it keeps its facet and its object. +const steps = [plugins.step("server", server as Host

), plugins.step("worker", worker as Host

)]; +const built = await plugins.build(plugins.state.value.dirty, context); +console.log( + "3. built:", + built.facets.map(({ plugin, facet, kind }) => `${plugin}/${facet}: ${kind}`), +); +const rolled = await plugins.rollout(steps, built, context); +for (const { name, report } of rolled.steps) { + console.log(` ${name}:`, JSON.stringify(report.plugins), "objects", JSON.stringify(report.objects)); +} +await until(() => notes.label.value === "v2"); +console.log("4. notes:", notes.label.value, "count", notes.count.value, "| index:", index.label.value); +console.log(" clock untouched:", clock.evaluatedAt.value === evaluatedAt); +console.log(" marks after the rollout:", JSON.stringify(plugins.state.value)); + +await watch.stop(); +await server.close(context); +await worker.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/clock/package.json b/packages/chord/test/next/examples/35-plugins/plugins/clock/package.json new file mode 100644 index 000000000..9cabc2e1a --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/clock/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/clock", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts b/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts new file mode 100644 index 000000000..5bc4b6ee3 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/clock/server.ts @@ -0,0 +1,9 @@ +// The clock plugin's server facet: serves `Clock` with the time its script was evaluated. The example never edits +// this plugin, so a reload leaves it as it is: same script, same facet, same object. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Clock } from "../contracts/contract.ts"; + +const evaluatedAt = new Date().toISOString(); + +export default defineFacet((host) => host.serve(Clock, () => ({ evaluatedAt: constant(evaluatedAt) }))); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/contracts/contract.ts b/packages/chord/test/next/examples/35-plugins/plugins/contracts/contract.ts new file mode 100644 index 000000000..cbab43b1e --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/contracts/contract.ts @@ -0,0 +1,21 @@ +// The contracts the plugins share: a contracts-only plugin (no facet file). A plugin that imports this file gets its +// own copy built into its script; the app's code declares the same contracts (same names, versions and shapes). + +import { hostKind, int, local, method, remote, str } from "@earendil-works/chord/next"; + +/** The worker's host kind: `Notes` lives there. */ +export const Worker = hostKind("example.worker"); + +/** Notes on the worker: a label, a count and `add`. */ +export const Notes = remote("example.notes", { + version: 1, + params: {}, + hostAddress: () => Worker(), + variants: { main: { state: { label: str(), count: int() }, methods: { add: method(undefined, int()) } } }, +}); + +/** The notes' label as the server shows it. */ +export const Index = local("example.index", { version: 1, params: {}, members: { label: str() } }); + +/** When the clock plugin's script was evaluated: it changes only when the plugin is reloaded. */ +export const Clock = local("example.clock", { version: 1, params: {}, members: { evaluatedAt: str() } }); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/contracts/package.json b/packages/chord/test/next/examples/35-plugins/plugins/contracts/package.json new file mode 100644 index 000000000..d1c133527 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/contracts/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/contracts", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/label.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/label.ts new file mode 100644 index 000000000..249f65179 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The label both of the plugin's facets show; the example edits it. + +export const label = "v1"; diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/package.json b/packages/chord/test/next/examples/35-plugins/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts new file mode 100644 index 000000000..ed3ded889 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/server.ts @@ -0,0 +1,7 @@ +// The notes plugin's server facet: serves `Index` with the same label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Index } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => host.serve(Index, () => ({ label: constant(label) }))); diff --git a/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts b/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts new file mode 100644 index 000000000..b4f4c8ae4 --- /dev/null +++ b/packages/chord/test/next/examples/35-plugins/plugins/notes/worker.ts @@ -0,0 +1,26 @@ +// The notes plugin's worker facet: serves `Notes`, its count kept in a document. + +import { constant, defineFacet, defineState, int, obj } from "@earendil-works/chord/next"; +import { Notes } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +const Count = defineState("count", { schema: obj({ n: int() }), version: 1, initial: () => ({ n: 0 }) }); + +export default defineFacet((host) => + host.serve(Notes, (_params, _principal, self) => { + const count = self.state(Count); + return { + variant: "main", + members: { + label: constant(label), + count: self.derive([count], ([value]) => value.n), + add: async (ctx) => { + await count.change((draft) => { + draft.n += 1; + }, ctx); + return count.value.n; + }, + }, + }; + }), +); diff --git a/packages/chord/test/next/examples/36-plugin-failures/main.ts b/packages/chord/test/next/examples/36-plugin-failures/main.ts new file mode 100644 index 000000000..1fe16092c --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/main.ts @@ -0,0 +1,156 @@ +// Plugin failures (slice 7c): every failure is per plugin. At startup a plugin whose setup throws is left out and the +// others run (a consumer of what it would serve waits). At a reload a plugin whose new code fails keeps its old code and +// is listed in the report's `failed`: a syntax error (its build failed), a module that throws while it is evaluated +// (`mapStack` names the plugin's file and line), a factory that throws while the host stages it. The closure rule: a +// provider and its consumer edited together, the provider broken, the consumer kept on its old code too (its new code +// needs the provider's new version) and still working; the next good edit applies both. Last, a reload whose context +// aborts after its commit: the commit stands, and the component keeps what was committed. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/36-plugin-failures/main.ts +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT, withAbortSignal } from "../../../../src/context/index.ts"; +import { int, local, str } from "../../../../src/next/contracts/index.ts"; +import { isChordError } from "../../../../src/next/data/index.ts"; +import { createHost, type ReloadReport } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, mapStack } from "../../../../src/next-node/plugins/index.ts"; +import { Weather } from "./plugins/weather/contract.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example edits files: it works on a copy of `plugins/`, removed at the end. +const root = await mkdtemp(join(tmpdir(), "chord-example-36-")); +await cp(fileURLToPath(new URL("./plugins/", import.meta.url)), root, { recursive: true }); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** Replaces `from` with `to` in the plugin file `file` (the copy). */ +async function edit(file: string, from: string, to: string): Promise { + const text = await readFile(join(root, file), "utf8"); + await writeFile(join(root, file), text.replace(from, to)); +} + +/** Waits until the watcher marks every plugin in `names` dirty, then builds the dirty plugins. */ +async function buildDirty(names: readonly string[]) { + await until(() => names.every((name) => plugins.state.value.dirty.includes(name))); + return plugins.build(plugins.state.value.dirty, context); +} + +/** A reload's report: what changed, its objects' fates, and each failed plugin with its error's mapped stack. */ +function print(report: ReloadReport): void { + console.log(" plugins:", JSON.stringify(report.plugins)); + const objects = report.objects.map((each) => `${each.object} ${each.kind}`); + if (objects.length > 0) console.log(" objects:", objects.join(", ")); + for (const { plugin, error } of report.failed) { + // The first line, and the second where it names the plugin's code: a build error, or a mapped stack frame. + const [message, next = ""] = mapStack(error).split("\n"); + const detail = !next.trimStart().startsWith("at ") || next.includes("/server.ts:") ? ` ${next.trim()}` : ""; + console.log(` failed ${plugin}: ${message}${detail}`); + } +} + +// The contracts the app reads (the plugins declare the same ones). +const Forecast = local("example.forecast", { version: 1, params: {}, members: { text: str() } }); +const Greeting = local("example.greeting", { version: 1, params: {}, members: { text: str() } }); +const Boot = local("example.boot", { version: 1, params: {}, members: { mode: str() } }); +const Counter = local("example.counter", { version: 1, params: {}, members: { start: int() } }); +const Summary = local("example.summary", { version: 1, params: {}, members: { text: str() } }); +const Status = local("example.status", { version: 1, params: {}, members: { text: str() } }); + +const plugins = createPlugins( + { + roots: [root], + kinds: ["server"], + hostPackages: { "@earendil-works/chord/next": chordNext }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); +const host = createHost({ onError: () => {} }); + +// 1. Startup: the weather plugin's setup throws; it is left out and reported, the other plugins run. A reference to +// what it would serve waits (`failed`, cause `waiting`: not served here), and so does its consumer's object. +const loaded = await plugins.load(host, { kind: "server", core: [] }, context); +for (const { plugin, error } of loaded.failed) console.log(`1. left out ${plugin}: ${(error as Error).message}`); +const weather = host.hold(Weather, {}, context); +const forecast = host.hold(Forecast, {}, context); +await until(() => forecast.state.value.kind === "failed"); +console.log(" weather:", JSON.stringify(weather.state.value)); +console.log(" forecast:", JSON.stringify(forecast.state.value)); +const greeting = await host.get(Greeting, {}, context); +const boot = await host.get(Boot, {}, context); +const counter = await host.get(Counter, {}, context); +const summary = await host.get(Summary, {}, context); +const status = await host.get(Status, {}, context); +const values = () => + [greeting.text.value, boot.mode.value, counter.start.value, summary.text.value] + .map((v) => JSON.stringify(v)) + .join(" "); +console.log(" running:", values()); + +// 2. Four bad edits at once. The greeter's label gets a syntax error; the boot module throws while it is evaluated; +// the counter's factory throws; the prices plugin moves its contract to version 2 and breaks its facet in the same +// edit, while the summary plugin (its copy of the contract now at version 2) builds fine. +await edit("greeter/label.ts", '"hello"', '"hello" +'); +await edit("boot/server.ts", "const Boot", 'throw new Error("boot: config missing");\n\nconst Boot'); +await edit("counter/server.ts", "() => ({ start: constant(1) })", '() => {\n\tthrow new Error("counter: no seed");\n}'); +await edit("prices/contract.ts", "version: 1", "version: 2"); +await edit("prices/server.ts", "constant(250)", "constant(250"); +const bad = await buildDirty(["@example/greeter", "@example/boot", "@example/counter", "@example/summary"]); +console.log("2. built:", bad.facets.map(({ plugin, kind }) => `${plugin}: ${kind}`).join(", ")); +// Each failed plugin keeps its old code; the summary's new code would ask for `Price` v2, which only the prices +// plugin's new (failed) code serves: the closure rule keeps the summary on its old code too. +print(await plugins.reload(host, bad, context)); +console.log(" running:", values()); +console.log(" still dirty:", JSON.stringify(plugins.state.value.dirty)); + +// 3. The fixes: the next build and reload apply every plugin, the prices provider and its consumer together. +await edit("greeter/label.ts", '"hello" +', '"hi"'); +await edit("boot/server.ts", 'throw new Error("boot: config missing");\n\n', ""); +await edit("boot/server.ts", 'constant("v1")', 'constant("v2")'); +await edit("counter/server.ts", '() => {\n\tthrow new Error("counter: no seed");\n}', "() => ({ start: constant(2) })"); +await edit("prices/server.ts", "constant(250", "constant(300)"); +const good = await buildDirty(["@example/greeter", "@example/boot", "@example/counter", "@example/prices"]); +console.log("3. fixed:"); +print(await plugins.reload(host, good, context)); +console.log(" running:", values()); +// The summary's new code waited for the new `Price` object (parked at the report): it runs once that one is live. +await until(() => summary.text.value.startsWith("price v2")); +console.log(" then:", values()); + +// 4. A reload whose context aborts after its commit, while it waits for its report: here the app shuts down the moment +// it sees the status object leave `live` (its new code waits for a new `Cache`, which takes 200 ms to open). The +// reload rejects `aborted`, but its commit stands: `details.committed` says what it applied, and the component's +// loaded set is what was committed (a later reload does not roll the cache plugin back). +console.log("4. status:", status.text.value); +await edit("cache/server.ts", 'const generation = "v1"', 'const generation = "v2"'); +const cache = await buildDirty(["@example/cache"]); +const abort = new AbortController(); +const watchStatus = await status.state.watch(context); +watchStatus.start(async (call) => { + if (call.kind !== "value") return; + console.log(" the status reference:", call.value.kind); + if (call.value.kind !== "live") abort.abort(new Error("the app shuts down")); +}); +try { + await plugins.reload(host, cache, withAbortSignal(abort.signal, context)); +} catch (error) { + if (!isChordError(error, "aborted")) throw error; + console.log(" rejected:", error.code, error.message, JSON.stringify(error.details)); +} +await until(() => status.text.value === "cache v2"); +console.log(" status:", status.text.value); +await edit("greeter/label.ts", '"hi"', '"hey"'); +print(await plugins.reload(host, await buildDirty(["@example/greeter"]), context)); +console.log(" after another reload:", status.text.value, "| greeting:", greeting.text.value); + +await watchStatus.stop(); +await host.close(context); +plugins.close(); +await rm(root, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/package.json new file mode 100644 index 000000000..2352aa044 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/boot", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/server.ts new file mode 100644 index 000000000..3dee3274f --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/boot/server.ts @@ -0,0 +1,7 @@ +// The boot plugin's server facet: serves `Boot`. The example makes this module throw while it is evaluated. + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; + +const Boot = local("example.boot", { version: 1, params: {}, members: { mode: str() } }); + +export default defineFacet((host) => host.serve(Boot, () => ({ mode: constant("v1") }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/package.json new file mode 100644 index 000000000..8403f6b07 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/cache", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/server.ts new file mode 100644 index 000000000..53fd0e247 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/cache/server.ts @@ -0,0 +1,27 @@ +// The cache plugin's server facet: `Cache` opens a store per generation, which takes a while (a native); `Status` reads +// the cache of the plugin's generation. A reload to a new generation swaps `Status` in place; its new code needs a new +// `Cache` object, and its first dependency check waits until that one is open: the reload's report waits for it too. + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; + +const generation = "v1"; + +const Cache = local("example.cache", { version: 1, params: { generation: str() }, members: { generation: str() } }); +const Status = local("example.status", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => { + host.serve(Cache, (params, self) => { + self.open( + async () => { + await new Promise((resolve) => setTimeout(resolve, 200)); + return {}; + }, + () => {}, + ); + return { generation: constant(params.generation) }; + }); + host.serve(Status, (_params, self) => { + const cache = self.get(Cache, { generation }); + return { text: self.derive([cache.generation], ([value]) => `cache ${value}`) }; + }); +}); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/package.json new file mode 100644 index 000000000..aa61f8c17 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/counter", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/server.ts new file mode 100644 index 000000000..08e2ea287 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/counter/server.ts @@ -0,0 +1,7 @@ +// The counter plugin's server facet: serves `Counter`. The example makes its factory throw. + +import { constant, defineFacet, int, local } from "@earendil-works/chord/next"; + +const Counter = local("example.counter", { version: 1, params: {}, members: { start: int() } }); + +export default defineFacet((host) => host.serve(Counter, () => ({ start: constant(1) }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/package.json new file mode 100644 index 000000000..04a79d9ce --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/forecast", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts new file mode 100644 index 000000000..54a92d46f --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/forecast/server.ts @@ -0,0 +1,13 @@ +// The forecast plugin's server facet: a consumer of the weather plugin's `Weather`. + +import { defineFacet, local, str } from "@earendil-works/chord/next"; +import { Weather } from "../weather/contract.ts"; + +const Forecast = local("example.forecast", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => + host.serve(Forecast, (_params, self) => { + const weather = self.get(Weather, {}); + return { text: self.derive([weather.celsius], ([celsius]) => `${celsius} degrees`) }; + }), +); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/label.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/label.ts new file mode 100644 index 000000000..bdfa4651d --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/label.ts @@ -0,0 +1,3 @@ +// The greeting's text; the example breaks it with a syntax error, then fixes it. + +export const greeting = "hello"; diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/package.json new file mode 100644 index 000000000..8b85f433a --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/greeter", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/server.ts new file mode 100644 index 000000000..14f719119 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/greeter/server.ts @@ -0,0 +1,8 @@ +// The greeter plugin's server facet: serves `Greeting` with the text of `label.ts`. + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; +import { greeting } from "./label.ts"; + +const Greeting = local("example.greeting", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Greeting, () => ({ text: constant(greeting) }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/contract.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/contract.ts new file mode 100644 index 000000000..701bedbbb --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/contract.ts @@ -0,0 +1,6 @@ +// The prices plugin's contract; the summary plugin imports (copies) it. The example moves it to version 2. + +import { int, local } from "@earendil-works/chord/next"; + +/** A price in cents. */ +export const Price = local("example.price", { version: 1, params: {}, members: { cents: int() } }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/package.json new file mode 100644 index 000000000..257de583d --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/prices", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/server.ts new file mode 100644 index 000000000..56655bc13 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/prices/server.ts @@ -0,0 +1,6 @@ +// The prices plugin's server facet: serves `Price`. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Price } from "./contract.ts"; + +export default defineFacet((host) => host.serve(Price, () => ({ cents: constant(250) }))); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/package.json new file mode 100644 index 000000000..6c54a9fa8 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/summary", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts new file mode 100644 index 000000000..3aad5f3e1 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/summary/server.ts @@ -0,0 +1,14 @@ +// The summary plugin's server facet: a consumer of the prices plugin's `Price`, at the version its copy of the +// contract declares. + +import { defineFacet, local, str } from "@earendil-works/chord/next"; +import { Price } from "../prices/contract.ts"; + +const Summary = local("example.summary", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => + host.serve(Summary, (_params, self) => { + const price = self.get(Price, {}); + return { text: self.derive([price.cents], ([cents]) => `price v${Price.version}: ${cents} cents`) }; + }), +); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/contract.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/contract.ts new file mode 100644 index 000000000..9798b45f5 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/contract.ts @@ -0,0 +1,6 @@ +// The weather plugin's contract; the forecast plugin imports (copies) it. + +import { int, local } from "@earendil-works/chord/next"; + +/** Today's temperature. */ +export const Weather = local("example.weather", { version: 1, params: {}, members: { celsius: int() } }); diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/package.json b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/package.json new file mode 100644 index 000000000..d72a57935 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/weather", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/server.ts b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/server.ts new file mode 100644 index 000000000..bbe12c2a7 --- /dev/null +++ b/packages/chord/test/next/examples/36-plugin-failures/plugins/weather/server.ts @@ -0,0 +1,12 @@ +// The weather plugin's server facet: its setup throws (no API key configured), so the host leaves the plugin out at +// startup and runs the others. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { Weather } from "./contract.ts"; + +const apiKey: string | undefined = undefined; + +export default defineFacet((host) => { + if (apiKey === undefined) throw new Error("weather: no API key configured"); + host.serve(Weather, () => ({ celsius: constant(21) })); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/app/index.ts b/packages/chord/test/next/examples/37-plugin-imports/app/index.ts new file mode 100644 index 000000000..b1182878a --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/app/index.ts @@ -0,0 +1,10 @@ +// The app's own code (base code): given to plugins as the host package `@example/app` (an ES module: a default export +// and named ones). A plugin imports it by that name, never by its path. + +/** The app's name and version. */ +export default { name: "example app", version: 3 }; + +/** Cents as a price. */ +export function currency(cents: number): string { + return `EUR ${(cents / 100).toFixed(2)}`; +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/app/legacy.cjs b/packages/chord/test/next/examples/37-plugin-imports/app/legacy.cjs new file mode 100644 index 000000000..a0c39787c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/app/legacy.cjs @@ -0,0 +1,7 @@ +// A CommonJS module of the app: given to plugins as the host package `@example/legacy`. + +module.exports = { + pad(text, width) { + return String(text).padStart(width, "."); + }, +}; diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/index.js b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/index.js new file mode 100644 index 000000000..c9e1272e1 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/index.js @@ -0,0 +1,6 @@ +// A small npm package (local, so the example needs no network): a plugin lists it in its `dependencies`, and the build +// inlines it into the plugin's script. + +export function greet(name) { + return `hello, ${name}`; +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json new file mode 100644 index 000000000..ae18d3b71 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-greeting/package.json @@ -0,0 +1,6 @@ +{ + "name": "example-greeting", + "version": "1.0.0", + "type": "module", + "main": "index.js" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/index.js b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/index.js new file mode 100644 index 000000000..ef13c18bc --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/index.js @@ -0,0 +1,4 @@ +// A package a plugin keeps external (`chord.external`: a native addon, a package that reads its own files): the build +// leaves its import as it is, and the facet's `require` loads it from the plugin's `node_modules` at run time. + +exports.where = () => __filename; diff --git a/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json new file mode 100644 index 000000000..c05122b96 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/deps/example-native/package.json @@ -0,0 +1,6 @@ +{ + "name": "example-native", + "version": "1.0.0", + "type": "commonjs", + "main": "index.js" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/fake-npm.mjs b/packages/chord/test/next/examples/37-plugin-imports/fake-npm.mjs new file mode 100644 index 000000000..d47f18b49 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/fake-npm.mjs @@ -0,0 +1,11 @@ +// A stand-in for `npm install` (the example needs no network): run in a plugin's directory, it copies every +// `file:` dependency of the plugin's package.json into its node_modules, as npm would install it. + +import { cpSync, readFileSync } from "node:fs"; +import { join, resolve } from "node:path"; + +const { dependencies = {} } = JSON.parse(readFileSync("package.json", "utf8")); +for (const [name, spec] of Object.entries(dependencies)) { + if (!spec.startsWith("file:")) throw new Error(`fake npm: ${name}@${spec} is not a local package`); + cpSync(resolve(spec.slice("file:".length)), join("node_modules", name), { recursive: true }); +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/main.ts b/packages/chord/test/next/examples/37-plugin-imports/main.ts new file mode 100644 index 000000000..8ed51b7d2 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/main.ts @@ -0,0 +1,76 @@ +// What a facet may import (slice 7c §2). The build inlines a plugin's own files, files of other plugins (a copied +// contract) and its npm `dependencies`; it leaves host packages (the app's instances: an ES module's default and named +// exports, a CommonJS module), `chord.external` packages and Node builtins external, answered when the script runs; a +// plugin's own files are read through `pluginFile`. Here (`plugins/`): the report plugin uses each of these; its +// dependencies are installed first (`installDependencies`; `fake-npm.mjs` stands in for npm: no network). The ruler +// plugin copies a file that imports a package, the refused plugin imports what a script may not, and the paths plugin +// uses `__dirname` and `__filename`: they are left out with their build errors. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/37-plugin-imports/main.ts +import { cp, mkdtemp, readdir, rm } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins } from "../../../../src/next-node/plugins/index.ts"; +import * as app from "./app/index.ts"; +import { Report } from "./plugins/shapes/contract.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example installs packages into a plugin: it works on a copy of `plugins/`, `app/` and `deps/` (the plugins' +// `file:` dependencies point there), removed at the end. +const copy = await mkdtemp(join(tmpdir(), "chord-example-37-")); +for (const dir of ["plugins", "app", "deps"]) { + await cp(fileURLToPath(new URL(`./${dir}/`, import.meta.url)), join(copy, dir), { recursive: true }); +} +const root = join(copy, "plugins"); + +// 1. The component. Its host packages: Chord, the app's ES module and a CommonJS module of the app. Its npm command +// (here a stand-in): what `installDependencies` runs in a plugin's directory. +const plugins = createPlugins( + { + roots: [root], + kinds: ["server"], + hostPackages: { + "@earendil-works/chord/next": chordNext, + "@example/app": app, + "@example/legacy": createRequire(import.meta.url)("./app/legacy.cjs"), + }, + install: { npmCommand: [process.execPath, fileURLToPath(new URL("./fake-npm.mjs", import.meta.url))] }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. Startup: every plugin whose build fails is left out with its errors (one per line: file, line, column, why). The +// report plugin's dependency is not installed yet. +const host = createHost({ onError: () => {} }); +const loaded = await plugins.load(host, { kind: "server", core: [] }, context); +console.log("1. left out at startup:"); +for (const { plugin, error } of loaded.failed) { + const [first, ...errors] = (error as Error).message.split("\n"); + console.log(` ${plugin}: ${first}`); + for (const line of errors) console.log(` ${line}`); +} + +// 3. The report plugin's dependencies, installed into its own node_modules; then it builds, and a reload adds it. +await plugins.installDependencies("@example/report", context); +console.log("2. installed:", (await readdir(join(root, "report", "node_modules"))).join(", ")); +const built = await plugins.build(["@example/report"], context); +for (const facet of built.facets) { + if (facet.kind !== "script") continue; + const files = facet.inputs.map(({ file }) => file.slice(root.length + 1)); + console.log(` built ${facet.plugin}/${facet.facet}: read ${files.join(", ")}; copied ${facet.copied.length} file`); +} +const report = await plugins.reload(host, built, context); +console.log(" reload:", JSON.stringify(report.plugins), "failed", report.failed.length); +const text = (await host.get(Report, {}, context)).text.value; +console.log("3. the report:", text); + +await host.close(context); +plugins.close(); +await rm(copy, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/package.json new file mode 100644 index 000000000..fc2e85a6f --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/paths", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/server.ts new file mode 100644 index 000000000..b3170c2a4 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/paths/server.ts @@ -0,0 +1,10 @@ +// The paths plugin's server facet: a script has no `__dirname` or `__filename` (it is evaluated with no file of its +// own); a file of the plugin is read through `pluginFile`. Each use is a build error. + +import { join } from "node:path"; +import { defineFacet } from "@earendil-works/chord/next"; + +export default defineFacet(() => { + console.log(join(__dirname, "data.json")); + console.log(__filename); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/other.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/other.ts new file mode 100644 index 000000000..fbb6085ae --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/other.ts @@ -0,0 +1,3 @@ +// A file of the refused plugin, imported dynamically by its facet (refused). + +export const other = 1; diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/package.json new file mode 100644 index 000000000..74f126ebd --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/refused", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/server.ts new file mode 100644 index 000000000..437d38020 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/refused/server.ts @@ -0,0 +1,11 @@ +// The refused plugin's server facet: imports a facet script may not have, each a build error with its file and line. +// A script is evaluated with no module identity: no `import.meta` (a file of the plugin is read through `pluginFile`), +// no literal dynamic `import()`; and base code is reached through a host package, not by its path. + +import { defineFacet } from "@earendil-works/chord/next"; +import app from "../../app/index.ts"; + +export default defineFacet(() => { + console.log(import.meta.url, app.name); + void import("./other.ts"); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/data/rates.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/data/rates.json new file mode 100644 index 000000000..ef32822cb --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/data/rates.json @@ -0,0 +1 @@ +{ "vat": 20 } diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/env.d.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/env.d.ts new file mode 100644 index 000000000..ac3f7fe53 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/env.d.ts @@ -0,0 +1,21 @@ +// Types for the report plugin's imports that its directory does not hold: the app's host packages (an app ships their +// types with its plugin template) and its dependencies before `npm install` (their own types once installed). + +declare module "@example/app" { + const app: { readonly name: string; readonly version: number }; + export default app; + export function currency(cents: number): string; +} + +declare module "@example/legacy" { + const legacy: { pad(text: string, width: number): string }; + export default legacy; +} + +declare module "example-greeting" { + export function greet(name: string): string; +} + +declare module "example-native" { + export function where(): string; +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/package.json new file mode 100644 index 000000000..751b92827 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/package.json @@ -0,0 +1,12 @@ +{ + "name": "@example/report", + "private": true, + "type": "module", + "dependencies": { + "example-greeting": "file:../../deps/example-greeting", + "example-native": "file:../../deps/example-native" + }, + "chord": { + "external": ["example-native"] + } +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts new file mode 100644 index 000000000..41201eeea --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/report/server.ts @@ -0,0 +1,32 @@ +// The report plugin's server facet: one of each import a facet may have. + +import { readFileSync } from "node:fs"; +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { pluginFile } from "@earendil-works/chord/next/plugins"; +// Host packages: an ES module (default and named imports) and a CommonJS one, the app's own instances. +import app, { currency } from "@example/app"; +import legacy from "@example/legacy"; +// An npm dependency: inlined into this script by the build (installed first). +import { greet } from "example-greeting"; +// A `chord.external` package: left external, loaded from this plugin's node_modules when the script runs. +import { where } from "example-native"; +// A contract of another plugin: a copy built into this script. +import { Report } from "../shapes/contract.ts"; + +// A file of this plugin, read at run time by its absolute path. +const rates = JSON.parse(readFileSync(pluginFile("data/rates.json"), "utf8")) as { readonly vat: number }; + +export default defineFacet((host) => + host.serve(Report, () => ({ + text: constant( + [ + `${app.name} v${app.version}`, + currency(1250), + legacy.pad("vat", 6), + `${rates.vat}%`, + greet("plugins"), + where().split("/").slice(-3).join("/"), + ].join(" | "), + ), + })), +); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/package.json new file mode 100644 index 000000000..49a09428c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/ruler", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts new file mode 100644 index 000000000..ec9a0dd74 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/ruler/server.ts @@ -0,0 +1,9 @@ +// The ruler plugin's server facet: imports a file of the shapes plugin that imports a package. Refused at build: a +// copied file may import only host packages and files of its own plugin. + +import { defineFacet } from "@earendil-works/chord/next"; +import { Millimetres } from "../shapes/units.ts"; + +export default defineFacet(() => { + console.log(Millimetres); +}); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/contract.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/contract.ts new file mode 100644 index 000000000..9dcc55e1c --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/contract.ts @@ -0,0 +1,7 @@ +// The shapes plugin's contract (a contracts-only plugin): other plugins import this file and get their own copy of it. +// A copied file may import host packages and files of its own plugin only. + +import { local, str } from "@earendil-works/chord/next"; + +/** A report: its lines. */ +export const Report = local("example.report", { version: 1, params: {}, members: { text: str() } }); diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/package.json b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/package.json new file mode 100644 index 000000000..8bedbd5da --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/shapes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts new file mode 100644 index 000000000..e2fc0bfb5 --- /dev/null +++ b/packages/chord/test/next/examples/37-plugin-imports/plugins/shapes/units.ts @@ -0,0 +1,7 @@ +// A file of the shapes plugin that imports a package (not a host package): fine for the shapes plugin itself, but a +// plugin importing this file would copy it, and a copied file may not import packages (the ruler plugin shows it). + +import { Type } from "typebox"; + +/** A length in millimetres. */ +export const Millimetres = Type.Number({ minimum: 0 }); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/app/config.ts b/packages/chord/test/next/examples/38-watch-and-rollout/app/config.ts new file mode 100644 index 000000000..c2c72bda8 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/app/config.ts @@ -0,0 +1,4 @@ +// The app's own code (base code): the plugins component watches it and marks `base` when it changes; Chord restarts +// nothing for it, the app restarts its hosts. + +export const appName = "example"; diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/package.json new file mode 100644 index 000000000..201740a8b --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/extra", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/server.ts new file mode 100644 index 000000000..dff218b90 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/incoming/extra/server.ts @@ -0,0 +1,8 @@ +// The extra plugin's server facet: not in plugins/ at first; the example copies the plugin in while the app runs. It +// declares its type itself (the same shape as `labelOf("extra")`). + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; + +const Extra = local("example.label.extra", { version: 1, params: {}, members: { text: str() } }); + +export default defineFacet((host) => host.serve(Extra, () => ({ text: constant("extra on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/main.ts b/packages/chord/test/next/examples/38-watch-and-rollout/main.ts new file mode 100644 index 000000000..7eeb92f1e --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/main.ts @@ -0,0 +1,150 @@ +// The watcher's marks and rollouts (slice 7c §5). The watcher marks what changed and never applies it: `dirty` for a +// plugin whose file changed (a contract file: every plugin that copies it), `install` for a changed `package.json`, +// `set` when a plugin directory or facet file is added or removed, `base` when the app's own code changes. The app +// builds the dirty plugins and rolls the build out over its hosts in order; a rollout clears the marks of the plugins +// it applied on every step. Here: a plugin added, one removed and one dropping its worker facet in one rollout; a +// rollout stopping at a step that rejects; a failed plugin staying dirty while the others clear; an edit during a build +// keeping its mark. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/38-watch-and-rollout/main.ts +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { createHost, type Host, type ReloadReport } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, type RolloutResult, type RolloutStep } from "../../../../src/next-node/plugins/index.ts"; + +const context = BACKGROUND_CONTEXT; + +// The example edits files: it works on a copy of `plugins/` and `app/`, removed at the end. +const copy = await mkdtemp(join(tmpdir(), "chord-example-38-")); +for (const dir of ["plugins", "app"]) { + await cp(fileURLToPath(new URL(`./${dir}/`, import.meta.url)), join(copy, dir), { recursive: true }); +} +const root = join(copy, "plugins"); + +/** Waits until `ready()` holds (the watcher's events arrive on their own time). */ +async function until(ready: () => boolean): Promise { + while (!ready()) await new Promise((resolve) => setTimeout(resolve, 10)); +} + +/** Replaces `from` with `to` in `file` (under the copy). */ +async function edit(file: string, from: string, to: string): Promise { + const text = await readFile(join(copy, file), "utf8"); + await writeFile(join(copy, file), text.replace(from, to)); +} + +/** Waits until every plugin of `names` is marked dirty. */ +const dirty = (...names: string[]) => until(() => names.every((name) => plugins.state.value.dirty.includes(name))); + +/** The marks, in one line. */ +const marks = () => JSON.stringify(plugins.state.value); + +/** What a step's report says, in one line. */ +const line = (report: ReloadReport) => + JSON.stringify({ ...report.plugins, failed: report.failed.map(({ plugin }) => plugin) }); + +/** A rollout, step by step. */ +function print(result: RolloutResult): void { + for (const { name, report } of result.steps) console.log(` ${name}:`, line(report)); + if (result.stopped !== undefined) { + console.log(` stopped at ${result.stopped.step}: ${(result.stopped.error as Error).message}`); + } +} + +const plugins = createPlugins( + { + roots: [root], + kinds: ["server", "worker"], + hostPackages: { "@earendil-works/chord/next": chordNext }, + // The app's own code: a change marks `base`. + base: [join(copy, "app")], + // A stand-in for npm (no network; these plugins have no dependencies). + install: { npmCommand: [process.execPath, "-e", ""] }, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); +const server = createHost({ onError: () => {} }); +const worker = createHost({ onError: () => {} }); +await plugins.load(server, { kind: "server", core: [] }, context); +await plugins.load(worker, { kind: "worker", core: [] }, context); +const steps = [plugins.step("server", server as Host), plugins.step("worker", worker as Host)]; +console.log("1. loaded:", plugins.discovery.plugins.map(({ name }) => name).join(", "), "| marks:", marks()); + +// 2. Each kind of change and its mark. +await edit("plugins/notes/label.ts", "notes v1", "notes v2"); +await dirty("@example/notes"); +console.log("2. a file of notes edited:", marks()); +await edit("plugins/contracts/contract.ts", "one type per plugin", "one type per plugin (edited)"); +await dirty("@example/contracts", "@example/tags"); +console.log(" the shared contract edited:", marks()); +await edit("plugins/tags/package.json", '"private": true', '"private": true,\n\t"description": "tags"'); +await until(() => plugins.state.value.install.includes("@example/tags")); +console.log(" tags' package.json edited:", marks()); +await plugins.installDependencies("@example/tags", context); +console.log(" tags installed:", marks()); +await cp(fileURLToPath(new URL("./incoming/extra/", import.meta.url)), join(root, "extra"), { recursive: true }); +await rm(join(root, "old"), { recursive: true }); +await rm(join(root, "tags", "worker.ts")); +await until(() => plugins.state.value.set); +// Deleting old's directory deleted its package.json too: an install mark, which goes with the next discovery. +console.log(" extra added, old removed, tags' worker facet deleted:", marks()); +await edit("app/config.ts", '"example"', '"example 2"'); +await until(() => plugins.state.value.base); +console.log(" the app's own code edited:", marks()); + +// 3. One build of the dirty plugins (discovery runs again first: `set`), rolled out server then worker. The server adds +// extra and removes old; the worker leaves tags out (it has no worker facet any more). The marks clear, but `base`: +// that one is the app's to act on (it restarts its hosts). +const built = await plugins.build(plugins.state.value.dirty, context); +console.log("3. built:", built.facets.map(({ plugin, facet }) => `${plugin}/${facet}`).join(", ")); +console.log(" removed:", JSON.stringify(built.removed)); +print(await plugins.rollout(steps, built, context)); +console.log(" marks:", marks()); + +// 4. A rollout stops at a step that rejects (a host the app cannot reach): the steps after it do not run, and the marks +// stay (the plugin was not applied everywhere). The next rollout clears them. +const unreachable: RolloutStep = { + name: "edge", + kinds: ["server"], + reload: async () => { + throw new Error("the edge host is unreachable"); + }, +}; +await edit("plugins/notes/label.ts", "notes v2", "notes v3"); +await dirty("@example/notes"); +const notes = await plugins.build(plugins.state.value.dirty, context); +console.log("4. rollout over server, edge, worker:"); +print(await plugins.rollout([steps[0]!, unreachable, steps[1]!], notes, context)); +console.log(" marks:", marks()); +print(await plugins.rollout(steps, notes, context)); +console.log(" marks after the next rollout:", marks()); + +// 5. A failed plugin stays dirty, the others clear: notes gets a syntax error, tags a good edit. +await edit("plugins/notes/label.ts", '"notes v3"', '"notes v4" +'); +await edit("plugins/tags/server.ts", "tags on the server", "tags v2 on the server"); +await dirty("@example/notes", "@example/tags"); +console.log("5. notes broken, tags edited:"); +print(await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context)); +console.log(" marks:", marks()); + +// 6. An edit after the build started keeps its mark: the rollout applies the build, but the newer edit is not in it. +await edit("plugins/notes/label.ts", '"notes v4" +', '"notes v4"'); +await dirty("@example/notes"); +const early = await plugins.build(plugins.state.value.dirty, context); +await edit("plugins/notes/label.ts", '"notes v4"', '"notes v5"'); +// The edit's event arrives on its own time (the mark is set already, so no change shows it). +await new Promise((resolve) => setTimeout(resolve, 200)); +console.log("6. built before the second edit:"); +print(await plugins.rollout(steps, early, context)); +console.log(" marks:", marks()); +print(await plugins.rollout(steps, await plugins.build(plugins.state.value.dirty, context), context)); +console.log(" marks after building again:", marks()); + +await server.close(context); +await worker.close(context); +plugins.close(); +await rm(copy, { recursive: true, force: true }); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/contract.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/contract.ts new file mode 100644 index 000000000..107b2fdbc --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/contract.ts @@ -0,0 +1,9 @@ +// The contracts the plugins share (a contracts-only plugin): every plugin importing this file has a copy of it, so an +// edit here marks every one of them dirty. + +import { local, str } from "@earendil-works/chord/next"; + +/** The label a plugin shows, one type per plugin. */ +export function labelOf(plugin: string) { + return local(`example.label.${plugin}`, { version: 1, params: {}, members: { text: str() } }); +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json new file mode 100644 index 000000000..d1c133527 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/contracts/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/contracts", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/label.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/label.ts new file mode 100644 index 000000000..9dd0e6455 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/label.ts @@ -0,0 +1,3 @@ +// The notes' label; the example edits it. + +export const label = "notes v1"; diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/package.json new file mode 100644 index 000000000..d59f5a427 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/notes", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts new file mode 100644 index 000000000..23a936014 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/server.ts @@ -0,0 +1,9 @@ +// The notes plugin's server facet: its label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(labelOf("notes"), () => ({ text: constant(`${label} on the server`) })), +); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts new file mode 100644 index 000000000..e01fc1159 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/notes/worker.ts @@ -0,0 +1,9 @@ +// The notes plugin's worker facet: its label. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; +import { label } from "./label.ts"; + +export default defineFacet((host) => + host.serve(labelOf("notes"), () => ({ text: constant(`${label} on the worker`) })), +); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/package.json new file mode 100644 index 000000000..2db7fdd46 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/old", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts new file mode 100644 index 000000000..afb0165f9 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/old/server.ts @@ -0,0 +1,6 @@ +// The old plugin's server facet; the example deletes the plugin's directory. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; + +export default defineFacet((host) => host.serve(labelOf("old"), () => ({ text: constant("old on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/package.json b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/package.json new file mode 100644 index 000000000..558ed9ed3 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/tags", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts new file mode 100644 index 000000000..8b8ce9d68 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/server.ts @@ -0,0 +1,6 @@ +// The tags plugin's server facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; + +export default defineFacet((host) => host.serve(labelOf("tags"), () => ({ text: constant("tags on the server") }))); diff --git a/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts new file mode 100644 index 000000000..7cbeb3ae4 --- /dev/null +++ b/packages/chord/test/next/examples/38-watch-and-rollout/plugins/tags/worker.ts @@ -0,0 +1,6 @@ +// The tags plugin's worker facet. + +import { constant, defineFacet } from "@earendil-works/chord/next"; +import { labelOf } from "../contracts/contract.ts"; + +export default defineFacet((host) => host.serve(labelOf("tags"), () => ({ text: constant("tags on the worker") }))); diff --git a/packages/chord/test/next/examples/39-browser-build/main.ts b/packages/chord/test/next/examples/39-browser-build/main.ts new file mode 100644 index 000000000..484951d89 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/main.ts @@ -0,0 +1,64 @@ +// A browser build (slice 7c §2): a facet of a browser kind (`browserKinds: ["web"]`) is built for the browser. Its +// stylesheets become one CSS file (`css`), an asset imported by URL (`./logo.svg?url`) a content-hashed file whose +// import evaluates to its URL under the app's `publicPath`. The build is plain data: script, stylesheet and assets, +// what the app serves to its pages (slice 11 ships it); a page evaluates the script with its host packages, as Node +// does here. A Node builtin is refused in a browser build. The example only reads `plugins/`: no copy. +// Run from packages/chord: +// node --conditions=source --experimental-strip-types test/next/examples/39-browser-build/main.ts +import { fileURLToPath } from "node:url"; +import { BACKGROUND_CONTEXT } from "../../../../src/context/index.ts"; +import { local, str } from "../../../../src/next/contracts/index.ts"; +import { createHost } from "../../../../src/next/host/index.ts"; +import * as chordNext from "../../../../src/next/index.ts"; +import { createPlugins, evaluateFacet } from "../../../../src/next-node/plugins/index.ts"; + +const context = BACKGROUND_CONTEXT; +const root = fileURLToPath(new URL("./plugins/", import.meta.url)); + +// 1. The component: `web` is a browser kind; its assets are served under `/plugin-assets/`. +const hostPackages = { "@earendil-works/chord/next": chordNext }; +const plugins = createPlugins( + { + roots: [root], + kinds: ["web"], + browserKinds: ["web"], + publicPath: "/plugin-assets/", + hostPackages, + onError: (error) => console.log("plugins:", error.kind), + }, + context, +); + +// 2. The first build: every discovered plugin. +const built = await plugins.build([], context); +const gallery = built.facets.find(({ plugin }) => plugin === "@example/gallery")!; +const bad = built.facets.find(({ plugin }) => plugin === "@example/bad-web")!; +if (gallery.kind !== "script" || bad.kind !== "failed") throw new Error("unexpected build results"); +console.log("1. @example/gallery/web: script of", gallery.source.length, "bytes, digest", gallery.digest.slice(0, 12)); +console.log(" stylesheet:", gallery.css?.name, JSON.stringify(gallery.css?.text.split("\n")[0])); +for (const asset of gallery.assets) console.log(" asset:", asset.name, `(${asset.bytes.length} bytes)`); +console.log(" the asset's URL in the script:", /\/plugin-assets\/[^"]+/.exec(gallery.source)?.[0]); +console.log(" read:", gallery.inputs.map(({ file }) => file.slice(root.length)).join(", ")); +console.log("2. @example/bad-web/web:", bad.errors.join("; ")); + +// 3. Plain data: a structured clone (what a message to a page or a worker carries) is the same build. +const shipped = structuredClone(built); +const copy = shipped.facets.find(({ plugin }) => plugin === "@example/gallery")!; +console.log( + "3. cloned:", + copy.kind === "script" && copy.digest === gallery.digest, + "| its plugins:", + Object.keys(shipped.plugins).join(", "), +); + +// 4. Where it runs (a page; Node stands in here), the script is evaluated with the host packages and loaded. +const Gallery = local("example.gallery", { version: 1, params: {}, members: { logo: str() } }); +if (copy.kind === "script") { + const facet = evaluateFacet(copy, { hostPackages, plugin: shipped.plugins["@example/gallery"]! }); + const page = createHost({ onError: () => {} }); + await page.load([{ name: "@example/gallery", facet }], context); + console.log("4. the gallery's logo on the page:", (await page.get(Gallery, {}, context)).logo.value); + await page.close(context); +} + +plugins.close(); diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/package.json b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/package.json new file mode 100644 index 000000000..6196d6fa7 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/bad-web", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/web.ts b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/web.ts new file mode 100644 index 000000000..8ae46e262 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/bad-web/web.ts @@ -0,0 +1,8 @@ +// The bad-web plugin's web facet: a Node builtin has no place in a browser build (refused at build). + +import { join } from "node:path"; +import { defineFacet } from "@earendil-works/chord/next"; + +export default defineFacet(() => { + console.log(join("a", "b")); +}); diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets.d.ts b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets.d.ts new file mode 100644 index 000000000..808a676b6 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets.d.ts @@ -0,0 +1,9 @@ +// Types for what a browser build imports besides code: a stylesheet (collected into the build's one CSS file) and an +// asset by URL (`./file.svg?url`: the URL the app serves it under, as a string). + +declare module "*.css" {} + +declare module "*?url" { + const url: string; + export default url; +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets/logo.svg b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets/logo.svg new file mode 100644 index 000000000..27de953fd --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/assets/logo.svg @@ -0,0 +1 @@ + diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/gallery.css b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/gallery.css new file mode 100644 index 000000000..cbc3f629a --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/gallery.css @@ -0,0 +1,8 @@ +.gallery { + display: grid; + gap: 8px; +} + +.gallery img { + width: 64px; +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/package.json b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/package.json new file mode 100644 index 000000000..618ee3124 --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/package.json @@ -0,0 +1,5 @@ +{ + "name": "@example/gallery", + "private": true, + "type": "module" +} diff --git a/packages/chord/test/next/examples/39-browser-build/plugins/gallery/web.ts b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/web.ts new file mode 100644 index 000000000..3755d990c --- /dev/null +++ b/packages/chord/test/next/examples/39-browser-build/plugins/gallery/web.ts @@ -0,0 +1,10 @@ +// The gallery plugin's web facet, built for a browser: its stylesheet is collected into one CSS file, its logo is an +// asset imported by URL (content-hashed, under the app's `publicPath`). + +import { constant, defineFacet, local, str } from "@earendil-works/chord/next"; +import "./gallery.css"; +import logo from "./assets/logo.svg?url"; + +const Gallery = local("example.gallery", { version: 1, params: {}, members: { logo: str() } }); + +export default defineFacet((host) => host.serve(Gallery, () => ({ logo: constant(logo) }))); diff --git a/packages/chord/test/next/host/facet.test.ts b/packages/chord/test/next/host/facet.test.ts index 0dd05e818..619cea025 100644 --- a/packages/chord/test/next/host/facet.test.ts +++ b/packages/chord/test/next/host/facet.test.ts @@ -1,4 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest"; +import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; import { int, local } from "../../../src/next/contracts/index.ts"; import { defineFacet, type Facet, type FacetHost, type FacetSelf, type Host } from "../../../src/next/host/index.ts"; import { defineRegistry } from "../../../src/next/registries/index.ts"; @@ -30,20 +31,23 @@ describe("facets", () => { it("contribute at most one value per registry, only while their setup runs", async () => { const Names = defineRegistry("app.names", { version: 1 }); - await expect( - loaded((_, self) => { - self.contribute(Names, "a"); - self.contribute(Names, "b"); - }), - ).rejects.toThrow("@test/app: registry app.names is contributed to twice"); + const twice = await loaded((_, self) => { + self.contribute(Names, "a"); + self.contribute(Names, "b"); + }); + expect(twice.failed.map(({ error }) => (error as Error).message)).toEqual([ + 'plugin "@test/app": @test/app: registry app.names is contributed to twice', + ]); + await twice.host.close(context); let later: FacetSelf | undefined; - await loaded((host, self) => { + const { host } = await loaded((host, self) => { host.serve(Names, { compose: (table) => table.length }); later = self; }); expect(() => later!.contribute(Names, "late")).toThrow( "@test/app: self.contribute is valid only in the facet's setup", ); + await host.close(context); }); it("are typed by exactly the app's principal type", () => { diff --git a/packages/chord/test/next/host/host.test.ts b/packages/chord/test/next/host/host.test.ts index f0a221b86..0ae2d5361 100644 --- a/packages/chord/test/next/host/host.test.ts +++ b/packages/chord/test/next/host/host.test.ts @@ -79,22 +79,46 @@ describe("load", () => { await host.close(context); }); - it("refuses invalid and duplicate plugin names, double serves and a second load", async () => { + it("leaves out and reports invalid and duplicate plugin names and double serves (the later); refuses a second load", async () => { const empty = defineFacet(() => {}); const info = defineFacet((host) => host.serve(Info, () => ({ title: constant("t") }))); - await expect(createHost(noErrors).load([{ name: "Bad Name", facet: empty }], context)).rejects.toThrow( - '"Bad Name" is not a valid plugin name', - ); + const loaded = async (facets: { readonly name: string; readonly facet: typeof empty }[]) => { + const host = createHost(noErrors); + const { failed } = await host.load(facets, context); + await host.close(context); + return failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + }; + expect(await loaded([{ name: "Bad Name", facet: empty }])).toEqual([ + ["Bad Name", '"Bad Name" is not a valid plugin name'], + ]); const twice = [ { name: "a", facet: empty }, { name: "a", facet: empty }, ]; - await expect(createHost(noErrors).load(twice, context)).rejects.toThrow('plugin "a" is loaded twice'); + expect(await loaded(twice)).toEqual([["a", 'plugin "a" is loaded twice']]); + const throwing = defineFacet(() => { + throw new Error("first failed"); + }); + // The first one's setup failing does not make the second one's name free (Sol). + const failing = createHost(noErrors); + const report = await failing.load( + [ + { name: "a", facet: throwing }, + { name: "a", facet: info }, + ], + context, + ); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + ["a", 'plugin "a": first failed'], + ["a", 'plugin "a" is loaded twice'], + ]); + expect(failing.hold(Info, {}, context).state.value).toMatchObject({ kind: "failed" }); + await failing.close(context); const served = [ { name: "a", facet: info }, { name: "b", facet: info }, ]; - await expect(createHost(noErrors).load(served, context)).rejects.toThrow("app.info is served twice"); + expect(await loaded(served)).toEqual([["b", 'plugin "b": app.info is served twice (plugin "a")']]); const host = createHost(noErrors); await host.load([], context); await expect(host.load([], context)).rejects.toThrow("load runs once per host"); @@ -807,14 +831,36 @@ forEachCodec(() => { const { server } = await linked((h) => serveAll(h, factories)); expect((await server.get(Who, { id: 1 }, context)).who.value).toBe("server"); const client = createHost

({ onError: () => {} }); - const loading = client.load([{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], context); - await expect(loading).rejects.toThrow("app.who: a host without a principal serves only local types"); + const report = await client.load( + [{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], + context, + ); + expect(report.failed).toEqual([ + { + plugin: "@test/app", + error: expect.objectContaining({ + message: 'plugin "@test/app": app.who: a host without a principal serves only local types', + }), + }, + ]); + await client.close(context); }); it("a host without an address serves no remote type, whatever its principal", async () => { const host = createHost

({ principal: { kind: "server" }, onError: () => {} }); - const loading = host.load([{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], context); - await expect(loading).rejects.toThrow("app.who: a host without an address serves only local types"); + const report = await host.load( + [{ name: "@test/app", facet: defineFacet

((h) => serveAll(h, [])) }], + context, + ); + expect(report.failed).toEqual([ + { + plugin: "@test/app", + error: expect.objectContaining({ + message: 'plugin "@test/app": app.who: a host without an address serves only local types', + }), + }, + ]); + await host.close(context); }); it("ctx.get acts as ctx.principal: a guest's call gets the guest's view; a local handler's acts as the host", async () => { @@ -1004,8 +1050,8 @@ describe("the selected provider's own checks for a request made here (fix round }); }); -describe("a load whose setup throws", () => { - it("closes the host as close does, links included, and rejects with the setup's error", async () => { +describe('a load whose setup throws (decision "Per plugin at startup and reload")', () => { + it("leaves that plugin out and reports it; the others publish, the host and its links stay open", async () => { const host = createHost({ principal: { kind: "server" }, onError: () => {} }); const [a, b] = memoryLinkPair(); const [c, d] = memoryLinkPair(); @@ -1013,17 +1059,31 @@ describe("a load whose setup throws", () => { host.connect(c, { principal: { kind: "server" }, codec: testCodec() }, context); const child = new RawPeer(b); const parent = new RawPeer(d); - const facet = defineFacet(() => { + const broken = defineFacet(() => { throw new Error("setup threw"); }); - await expect(host.load([{ name: "@test/app", facet }], context)).rejects.toThrow("setup threw"); + const report = await host.load( + [ + { name: "@test/broken", facet: broken }, + { + name: "@test/app", + facet: defineFacet((h) => h.serve(Info, () => ({ title: constant("t") }))), + }, + ], + context, + ); + expect(report.failed).toEqual([ + { plugin: "@test/broken", error: expect.objectContaining({ message: 'plugin "@test/broken": setup threw' }) }, + ]); await transported(); - expect([child.closed, parent.closed]).toEqual([true, true]); - expect(await rejection(host.get(Page, { id: 1 }, context))).toEqual({ - code: "released", - reason: "closed", - message: "the host is closed", + expect([child.closed, parent.closed]).toEqual([false, false]); + expect((await host.get(Info, {}, context)).title.value).toBe("t"); + // What the left-out plugin would have served waits as any missing type does. + expect(host.hold(Page, { id: 1 }, context).state.value).toMatchObject({ + kind: "failed", + cause: { kind: "waiting" }, }); + await host.close(context); }); }); @@ -1397,7 +1457,7 @@ describe("reload: the API (slice 7b §1)", () => { await host.close(context); }); - it("an abort before the commit rejects aborted and changes nothing; after it, only ends the wait", async () => { + it("an abort before the commit rejects aborted and changes nothing", async () => { const host = createHost(noErrors); await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); const { context: aborted, cancel } = withCancel(context); @@ -1409,31 +1469,74 @@ describe("reload: the API (slice 7b §1)", () => { await host.close(context); }); - it("rejects with a setup's or a factory's throw, naming the plugin, and the host stays on its old code", async () => { + it("an abort after the commit rejects aborted naming what the commit applied, which stands", async () => { + const host = createHost(noErrors); + await host.load( + [ + { name: "@app/info", facet: infoFacet("a") }, + { name: "@app/gone", facet: defineFacet(() => {}) }, + ], + context, + ); + const info = await host.get(Info, {}, context); + const { context: aborting, cancel } = withCancel(context); + // The abort lands right after the commit (its publication batch), while the reload waits for its report. + const { batch } = host as ChordHost; + const commit = batch.commit.bind(batch); + const hook = vi.spyOn(batch, "commit").mockImplementation((adopt, ctx) => { + commit(adopt, ctx); + if (ctx === aborting) cancel(new Error("stop")); + }); + const reloading = host.reload( + [ + { name: "@app/info", facet: infoFacet("b") }, + { name: "@app/new", facet: defineFacet(() => {}) }, + ], + aborting, + ); + const error = await rejection(reloading); + hook.mockRestore(); + expect(error).toMatchObject({ + code: "aborted", + details: { committed: { added: ["@app/new"], changed: ["@app/info"], removed: ["@app/gone"], failed: [] } }, + }); + expect(info.title.value).toBe("b"); + await host.close(context); + }); + + it("reports a setup's or a factory's throw as failed, naming the plugin, which stays on its old code", async () => { const host = createHost(noErrors); await host.load([{ name: "@app/info", facet: infoFacet("a") }], context); const info = await host.get(Info, {}, context); const throwing = defineFacet(() => { throw new Error("setup threw"); }); - expect(await rejection(host.reload([{ name: "@app/info", facet: throwing }], context))).toMatchObject({ - message: 'plugin "@app/info": setup threw', - }); + const failedOf = async (facets: { readonly name: string; readonly facet: typeof throwing }[]) => { + const report = await host.reload(facets, context); + return report.failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + }; + expect(await failedOf([{ name: "@app/info", facet: throwing }])).toEqual([ + ["@app/info", 'plugin "@app/info": setup threw'], + ]); const factoryThrows = defineFacet((h) => h.serve(Info, () => { throw new Error("factory threw"); }), ); - expect(await rejection(host.reload([{ name: "@app/info", facet: factoryThrows }], context))).toMatchObject({ - message: 'plugin "@app/info", app.info: factory threw', - }); - // A served-twice set and a broken accepts table are staging errors too. + expect(await failedOf([{ name: "@app/info", facet: factoryThrows }])).toEqual([ + ["@app/info", 'plugin "@app/info", app.info: factory threw'], + ]); + // A type another plugin serves already fails the plugin with new code; an added one is left out. const twice = [ { name: "@app/info", facet: infoFacet("b") }, { name: "@app/other", facet: infoFacet("c") }, ]; - expect(await rejection(host.reload(twice, context))).toMatchObject({ message: "app.info is served twice" }); - expect(info.title.value).toBe("a"); + const report = await host.reload(twice, context); + expect(report.plugins).toEqual({ added: [], changed: ["@app/info"], removed: [] }); + expect(report.failed.map(({ plugin, error }) => [plugin, (error as Error).message])).toEqual([ + ["@app/other", 'plugin "@app/other": app.info is served twice (plugin "@app/info")'], + ]); + expect(info.title.value).toBe("b"); expect(info.state.value).toEqual({ kind: "live" }); await host.close(context); }); diff --git a/packages/chord/test/next/host/object.test.ts b/packages/chord/test/next/host/object.test.ts index 1f1fb06d4..d177122ea 100644 --- a/packages/chord/test/next/host/object.test.ts +++ b/packages/chord/test/next/host/object.test.ts @@ -1298,6 +1298,7 @@ describe("reload: fates (slice 7b §4)", () => { const report = await host.reload([], context); expect(report).toEqual({ plugins: { added: [], changed: [], removed: ["@test/app"] }, + failed: [], objects: [{ object: "app.pure", kind: "ended" }], }); expect(pure.state.value).toMatchObject({ @@ -1661,7 +1662,7 @@ describe("reload: GC", () => { const objects = instancesOf(ChordObject, "remove"); const host = createHost({ onError: () => {} }); /** Loads the plugin; only the host keeps its facet. */ - const load = (): [WeakRef, Promise] => { + const load = (): [WeakRef, Promise] => { const made = defineFacet((h, self) => { servePure(h, "v1", 1); h.serve(Names, { compose: (table) => table.map(({ value }) => value) }); diff --git a/packages/chord/test/next/host/staging.test.ts b/packages/chord/test/next/host/staging.test.ts index 9606e05f7..b0b01b563 100644 --- a/packages/chord/test/next/host/staging.test.ts +++ b/packages/chord/test/next/host/staging.test.ts @@ -1,17 +1,25 @@ -// Tests of staging (slice 7b §2): a reload's setups and recorded factories; each kind of throw leaves the host on its -// old code; recorded declarations decide an object's fate. +// Tests of staging (slice 7b §2, per plugin since slice 7c): a reload's setups and recorded factories; each kind of +// throw keeps the failing plugin on its old code, reported; recorded declarations decide an object's fate. import { describe, expect, it } from "vitest"; import { BACKGROUND_CONTEXT as context } from "../../../src/context/index.ts"; -import { int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; +import { hostKind, int, local, method, obj, remote, str } from "../../../src/next/contracts/index.ts"; import { defineState } from "../../../src/next/documents/index.ts"; -import { createHost, defineFacet, type FacetHost, type StoreSelf } from "../../../src/next/host/index.ts"; +import { + createHost, + defineFacet, + type FacetHost, + type Host, + type PluginFacet, + type PluginFailure, + type StoreSelf, +} from "../../../src/next/host/index.ts"; import { defineJob } from "../../../src/next/jobs/index.ts"; import { defineRegistry } from "../../../src/next/registries/index.ts"; import { constant } from "../../../src/next/state/index.ts"; import { defineTable } from "../../../src/next/store/table.ts"; import { evolveTable } from "../../../src/next/store/table-evolve.ts"; -import { flushed, memoryStore, rejection, Server } from "../support.ts"; +import { flushed, memoryStore, Server } from "../support.ts"; const Store = local("app.store", { version: 1, params: {}, members: { label: str(), bump: method(undefined, int()) } }); @@ -25,7 +33,14 @@ function serveStore(label: string, declare: (self: StoreSelf) => void = () => {} ); } -describe("staging: a throw leaves the host on its old code", () => { +/** The error of the one plugin `facets`' reload reports failed (it kept its old code). */ +async function failedReload(host: Host, facets: readonly PluginFacet[]): Promise { + const { failed } = await host.reload(facets, context); + expect(failed).toHaveLength(1); + return failed[0]!.error as Error; +} + +describe("staging: a throw keeps the plugin on its old code, reported", () => { const throwing: [string, ReturnType, string][] = [ [ "a setup", @@ -67,7 +82,7 @@ describe("staging: a throw leaves the host on its old code", () => { ], ]; for (const [what, facet, message] of throwing) { - it(`${what}: the reload rejects with it, nothing changed`, async () => { + it(`${what}: the reload reports it, the plugin unchanged`, async () => { const host = createHost({ onError: () => {} }); const v2 = defineState("doc", { schema: obj({ n: int() }), @@ -77,8 +92,7 @@ describe("staging: a throw leaves the host on its old code", () => { }); await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => self.state(v2)) }], context); const store = await host.get(Store, {}, context); - const error = await rejection(host.reload([{ name: "@test/app", facet }], context)); - expect((error as Error).message).toContain(message); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain(message); expect(store.label.value).toBe("v1"); expect(store.state.value).toEqual({ kind: "live" }); // The plugin is still the old one: reloading it with the same facet set again is a change of nothing. @@ -96,9 +110,9 @@ describe("staging: a throw leaves the host on its old code", () => { const host = createHost({ principal: { kind: "server" }, onError: () => {} }); await host.load([], context); const facet = defineFacet((h) => h.serve(Remote, () => ({ variant: "v", members: { n: constant(1) } }))); - expect(await rejection(host.reload([{ name: "@test/app", facet }], context))).toMatchObject({ - message: expect.stringContaining("a host without an address serves only local types"), - }); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain( + "a host without an address serves only local types", + ); await host.close(context); }); }); @@ -157,9 +171,9 @@ describe("staging: recorded declarations decide the fate", () => { await host.load([{ name: "@test/app", facet: serveStore("v1", (self) => void self.job(Job1, run)) }], context); await host.get(Store, {}, context); const facet = serveStore("v2", (self) => void self.job(Job1b, run)); - expect(await rejection(host.reload([{ name: "@test/app", facet }], context))).toMatchObject({ - message: expect.stringContaining("schema changed without a version bump"), - }); + expect((await failedReload(host, [{ name: "@test/app", facet }])).message).toContain( + "schema changed without a version bump", + ); await host.close(context); }); @@ -176,15 +190,15 @@ describe("staging: recorded declarations decide the fate", () => { ); await host.load([{ name: "@test/app", facet: page(false) }], context); await host.get(Page, { id: 1 }, context); - expect(await rejection(host.reload([{ name: "@test/app", facet: page(true) }], context))).toMatchObject({ - message: expect.stringContaining("self.contribute is valid only in a local type with empty params"), - }); + expect((await failedReload(host, [{ name: "@test/app", facet: page(true) }])).message).toContain( + "self.contribute is valid only in a local type with empty params", + ); await host.close(context); }); }); describe("staging: declaration parity with an attempt (fix round 1)", () => { - it("rejects retryOn of an attempt-owned writable without changing the live host", async () => { + it("refuses retryOn of an attempt-owned writable, the plugin kept on its old code", async () => { const T = local("astra.staging", { version: 1, params: {}, members: { n: int() } }); const oldFacet = defineFacet((h) => h.serve(T, () => ({ n: constant(1) }))); const newFacet = defineFacet((h) => @@ -196,15 +210,307 @@ describe("staging: declaration parity with an attempt (fix round 1)", () => { const host = createHost({ onError: () => {} }); await host.load([{ name: "@astra/staging", facet: oldFacet }], context); const old = await host.get(T, {}, context); - let rejected = false; - try { - await host.reload([{ name: "@astra/staging", facet: newFacet }], context); - } catch { - rejected = true; - } + const { failed } = await host.reload([{ name: "@astra/staging", facet: newFacet }], context); const state = old.state.value; old.release(); await host.close(context); - expect({ rejected, state: state.kind }).toEqual({ rejected: true, state: "live" }); + expect({ failed: failed.map(({ plugin }) => plugin), state: state.kind }).toEqual({ + failed: ["@astra/staging"], + state: "live", + }); + }); +}); + +describe("staging per plugin (slice 7c §4)", () => { + const NotesV1 = local("app.notes", { version: 1, params: {}, members: { count: int() } }); + const NotesV2 = local("app.notes", { version: 2, params: {}, members: { count: int(), title: str() } }); + const View = local("app.view", { version: 1, params: {}, members: { label: str() } }); + const Extra = local("app.extra", { version: 1, params: {}, members: { n: int() } }); + + /** The notes provider at `version`, labelled through its count. */ + const notes = (version: 1 | 2, count: number) => + defineFacet((h) => + version === 1 + ? h.serve(NotesV1, () => ({ count: constant(count) })) + : h.serve(NotesV2, () => ({ count: constant(count), title: constant("t") })), + ); + /** A consumer of the notes at `version`, labelled `label` and the count it reads. */ + const view = (version: 1 | 2, label: string) => + defineFacet((h) => + h.serve(View, (_, self) => { + const count = (version === 1 ? self.get(NotesV1, {}) : self.get(NotesV2, {})).count; + return { label: self.derive([count], ([n]) => `${label} ${n}`) }; + }), + ); + const throwing = defineFacet(() => { + throw new Error("setup threw"); + }); + const failedOf = (report: { readonly failed: readonly PluginFailure[] }) => + report.failed.map(({ plugin, error }) => [plugin, (error as Error).message]); + + it("a failing plugin keeps its old code; the others of the reload apply", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/store", facet: serveStore("v1") }, + ], + context, + ); + const store = await host.get(Store, {}, context); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: throwing }, + { name: "@test/store", facet: serveStore("v2") }, + ], + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/store"], removed: [] }); + expect(failedOf(report)).toEqual([["@test/notes", 'plugin "@test/notes": setup threw']]); + expect(report.objects).toEqual([{ object: "app.store", kind: "swapped" }]); + await flushed(); + expect([store.label.value, counted.count.value, counted.state.value]).toEqual(["v2", 1, { kind: "live" }]); + // Fixed later, it applies as any change does. + const fixed = await host.reload( + [ + { name: "@test/notes", facet: notes(1, 2) }, + { name: "@test/store", facet: serveStore("v2") }, + ], + context, + ); + expect([fixed.plugins.changed, fixed.failed]).toEqual([["@test/notes", "@test/store"], []]); + await host.close(context); + }); + + it("a factory throwing in staging fails its plugin alone; an added plugin that fails is left out", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/store", facet: serveStore("v1") }, + ], + context, + ); + await host.get(Store, {}, context); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/extra", facet: throwing }, + { name: "@test/notes", facet: notes(1, 2) }, + { + name: "@test/store", + facet: serveStore("v2", () => { + throw new Error("factory threw"); + }), + }, + ], + context, + ); + expect(report.plugins).toEqual({ added: [], changed: ["@test/notes"], removed: [] }); + expect(failedOf(report)).toEqual([ + ["@test/extra", 'plugin "@test/extra": setup threw'], + ["@test/store", 'plugin "@test/store", app.store: factory threw'], + ]); + await flushed(); + expect(counted.count.value).toBe(2); + await host.close(context); + }); + + it("the closure rule: a consumer edited together with a failing provider keeps its old code", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: throwing }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect(report.plugins.changed).toEqual([]); + expect(failedOf(report)).toEqual([ + ["@test/notes", 'plugin "@test/notes": setup threw'], + ["@test/view", 'plugin "@test/view": needs app.notes v2, which plugin "@test/notes" serves on its old code'], + ]); + expect(report.objects).toEqual([]); + await flushed(); + expect([shown.label.value, shown.state.value]).toEqual(["old 1", { kind: "live" }]); + // Both fixed together: both apply. + const both = await host.reload( + [ + { name: "@test/notes", facet: notes(2, 5) }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect([both.plugins.changed, both.failed]).toEqual([["@test/notes", "@test/view"], []]); + await flushed(); + expect(shown.label.value).toBe("new 5"); + await host.close(context); + }); + + it("takes the plugins whose new code failed before staging (`failed`): the closure rule and the report see them", async () => { + const host = createHost({ onError: () => {} }); + const provider = notes(1, 1); + await host.load( + [ + { name: "@test/notes", facet: provider }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + // Its new build failed: the caller passes its loaded facet, and says so. + const before = [{ plugin: "@test/notes", error: new Error("build failed") }]; + const report = await host.reload( + [ + { name: "@test/notes", facet: provider }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + { failed: before }, + ); + expect(failedOf(report)).toEqual([ + ["@test/notes", "build failed"], + ["@test/view", 'plugin "@test/view": needs app.notes v2, which plugin "@test/notes" serves on its old code'], + ]); + expect([report.plugins.changed, shown.label.value]).toEqual([[], "old 1"]); + await host.close(context); + }); + + it("the closure rule checks a request as a real bind does, the kept provider's host address included (Astra)", async () => { + const Here = hostKind("app.here"); + const Elsewhere = hostKind("app.elsewhere"); + const placed = (only: boolean) => + remote("app.provider", { + version: 1, + params: { id: int() }, + hostAddress: ({ id }) => (only && id !== 1 ? Elsewhere() : Here()), + variants: { main: { state: { n: int() } } }, + }); + const Consumer = local("app.consumer", { version: 1, params: {}, members: { n: int() } }); + const consumer = (id: number) => + defineFacet((h) => + h.serve(Consumer, (_, self) => ({ n: self.derive([self.get(placed(false), { id }).n], ([n]) => n) })), + ); + const host = createHost({ address: Here(), principal: { kind: "owner" }, onError: () => {} }); + await host.load( + [ + // The provider's own code places every id but 1 on another host. + { + name: "@test/provider", + facet: defineFacet((h) => + h.serve(placed(true), ({ id }) => ({ variant: "main", members: { n: constant(id) } })), + ), + }, + { name: "@test/consumer", facet: consumer(1) }, + ], + context, + ); + const shown = await host.get(Consumer, {}, context); + const report = await host.reload( + [ + { name: "@test/provider", facet: throwing }, + { name: "@test/consumer", facet: consumer(2) }, + ], + context, + ); + expect(report.failed.map(({ plugin }) => plugin)).toEqual(["@test/provider", "@test/consumer"]); + expect([shown.state.value, shown.n.value]).toEqual([{ kind: "live" }, 1]); + await host.close(context); + }); + + it("the closure rule covers a name nobody serves while a plugin failed, and runs to a fixpoint", async () => { + const Middle = local("app.middle", { version: 2, params: {}, members: { n: int() } }); + const Top = local("app.top", { version: 1, params: {}, members: { n: int() } }); + const host = createHost({ onError: () => {} }); + const middle = (needs: boolean) => + defineFacet((h) => h.serve(Middle, (_, self) => ({ n: needs ? self.get(Extra, {}).n : constant(0) }))); + const top = (needs: boolean) => + defineFacet((h) => h.serve(Top, (_, self) => ({ n: needs ? self.get(Middle, {}).n : constant(0) }))); + await host.load( + [ + { name: "@test/middle", facet: middle(false) }, + { name: "@test/top", facet: top(false) }, + ], + context, + ); + await host.get(Top, {}, context); + await host.get(Middle, {}, context); + // The new extra plugin fails: the middle's new code needs Extra, served by nobody now; the top's new code needs + // the middle, which keeps its old code (another version is not involved: its request binds to the old one). + const report = await host.reload( + [ + { name: "@test/extra", facet: throwing }, + { name: "@test/middle", facet: middle(true) }, + { name: "@test/top", facet: top(true) }, + ], + context, + ); + expect(failedOf(report)).toEqual([ + ["@test/extra", 'plugin "@test/extra": setup threw'], + ["@test/middle", 'plugin "@test/middle": needs app.extra v1, which no plugin serves now (a plugin failed)'], + ]); + expect(report.plugins.changed).toEqual(["@test/top"]); + await host.close(context); + }); + + it("without a failing plugin, a request that does not bind is an ordinary wait: the consumer applies and parks", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(1, "old") }, + ], + context, + ); + const shown = await host.get(View, {}, context); + const report = await host.reload( + [ + { name: "@test/notes", facet: notes(1, 1) }, + { name: "@test/view", facet: view(2, "new") }, + ], + context, + ); + expect([report.plugins.changed, report.failed]).toEqual([["@test/notes", "@test/view"], []]); + expect(report.objects).toEqual([ + { object: "app.notes", kind: "swapped" }, + expect.objectContaining({ object: "app.view", kind: "parked" }), + ]); + await flushed(); + expect(shown.state.value).toMatchObject({ kind: "failed", cause: { kind: "waiting" } }); + await host.close(context); + }); + + it("a type moving between plugins while its old owner fails: the new owner collides and keeps its old code too", async () => { + const host = createHost({ onError: () => {} }); + await host.load( + [ + { name: "@test/a", facet: notes(1, 1) }, + { name: "@test/b", facet: defineFacet(() => {}) }, + ], + context, + ); + const counted = await host.get(NotesV1, {}, context); + const report = await host.reload( + [ + { name: "@test/a", facet: throwing }, + { name: "@test/b", facet: notes(1, 2) }, + ], + context, + ); + expect(failedOf(report)).toEqual([ + ["@test/a", 'plugin "@test/a": setup threw'], + ["@test/b", 'plugin "@test/b": app.notes is served twice (plugin "@test/a")'], + ]); + expect(counted.count.value).toBe(1); + await host.close(context); }); }); diff --git a/packages/chord/test/next/registries/registry.test.ts b/packages/chord/test/next/registries/registry.test.ts index 61db2cd14..f1d31373e 100644 --- a/packages/chord/test/next/registries/registry.test.ts +++ b/packages/chord/test/next/registries/registry.test.ts @@ -19,24 +19,36 @@ describe("tokens", () => { describe("providers", () => { const Names = defineRegistry("app.names", { version: 1 }); - it("one per host: a second serve is refused at load", async () => { + it("one per host: a second serve leaves its plugin out at load, reported", async () => { const host = createHost({ onError: () => {} }); const serve = defineFacet((h) => h.serve(Names, { compose: (table) => table.length })); - await expect( - host.load( - [ - { name: "@app/a", facet: serve }, - { name: "@app/b", facet: serve }, - ], - context, - ), - ).rejects.toThrow("registry app.names is served twice"); + const { failed } = await host.load( + [ + { name: "@app/a", facet: serve }, + { name: "@app/b", facet: serve }, + ], + context, + ); + expect(failed).toEqual([ + { + plugin: "@app/b", + error: expect.objectContaining({ + message: 'plugin "@app/b": registry app.names is served twice (plugin "@app/a")', + }), + }, + ]); + await host.close(context); }); - it("a provider without compose is refused at load (plugin code is not type-checked)", async () => { - await expect(loadedPlugins({ "@app/a": (h) => h.serve(Names, {} as never) })).rejects.toThrow( - "registry app.names: compose is a function", - ); + it("a provider without compose leaves its plugin out at load (plugin code is not type-checked)", async () => { + const { host, failed } = await loadedPlugins({ "@app/a": (h) => h.serve(Names, {} as never) }); + expect(failed).toEqual([ + { + plugin: "@app/a", + error: expect.objectContaining({ message: 'plugin "@app/a": registry app.names: compose is a function' }), + }, + ]); + await host.close(context); }); it("a registry whose provider comes with `load` heals a `host.registry` made before it", async () => { diff --git a/packages/chord/test/next/support.ts b/packages/chord/test/next/support.ts index 06fb043ac..7037ab438 100644 --- a/packages/chord/test/next/support.ts +++ b/packages/chord/test/next/support.ts @@ -38,6 +38,7 @@ import { type FacetSelf, type Host, type HostError, + type PluginFailure, type RefState, type Self, type StoreSelf, @@ -240,12 +241,13 @@ export async function loaded( readonly readyTimeout?: number; readonly store?: Store; } = {}, -): Promise<{ readonly host: Host; readonly errors: HostError[] }> { +): Promise<{ readonly host: Host; readonly errors: HostError[]; readonly failed: readonly PluginFailure[] }> { const errors: HostError[] = []; const { telemetry, closeTimeout, readyTimeout, store } = options; const host = createHost({ telemetry, closeTimeout, readyTimeout, store, onError: (error) => errors.push(error) }); - await host.load([{ name: "@test/app", facet: defineFacet(setup) }], options.context ?? BACKGROUND_CONTEXT); - return { host, errors }; + const facets = [{ name: "@test/app", facet: defineFacet(setup) }]; + const { failed } = await host.load(facets, options.context ?? BACKGROUND_CONTEXT); + return { host, errors, failed }; } /** @@ -255,12 +257,12 @@ export async function loaded( export async function loadedPlugins( setups: Readonly void>>, options: { readonly telemetry?: InMemoryTelemetryContext; readonly readyTimeout?: number } = {}, -): Promise<{ readonly host: Host; readonly errors: HostError[] }> { +): Promise<{ readonly host: Host; readonly errors: HostError[]; readonly failed: readonly PluginFailure[] }> { const errors: HostError[] = []; const host = createHost({ ...options, onError: (error) => errors.push(error) }); const facets = Object.entries(setups).map(([name, setup]) => ({ name, facet: defineFacet(setup) })); - await host.load(facets, BACKGROUND_CONTEXT); - return { host, errors }; + const { failed } = await host.load(facets, BACKGROUND_CONTEXT); + return { host, errors, failed }; } /** A store on a fresh SQLite `:memory:` database, as Chord's layers see it. */ diff --git a/tsconfig.json b/tsconfig.json index eea4c716b..b49e773ff 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,6 +9,7 @@ "@earendil-works/chord/context": ["./packages/chord/src/context/index.ts"], "@earendil-works/chord/delta": ["./packages/chord/src/delta/index.ts"], "@earendil-works/chord/next": ["./packages/chord/src/next/index.ts"], + "@earendil-works/chord/next/plugins": ["./packages/chord/src/next-node/plugins/index.ts"], "@earendil-works/chord/node": ["./packages/chord/src/node.ts"], "@earendil-works/chord/output": ["./packages/chord/src/output.ts"], "@earendil-works/chord/telemetry-sqlite": ["./packages/chord/src/telemetry-sqlite/index.ts"],