From 7dc38ff0e9490483b368c2b14531c9199c4cbee1 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 3 Nov 2026 16:48:38 +0100 Subject: [PATCH] feat(p2): MCP servers on the host, on environments and from projects A manager on the p2 server owns every MCP connection and process. Session workers hold one view each and lease the locations their conversations work at; the manager runs the owner's mcp.json servers there (stdio per session and location, or shared; HTTP shared by default) and offers their tools as deferred Durable tools found through tool_search. Stdio servers on an environment start through its pi-env daemon, one connection per environment shared by its instances, through cmd.exe on Windows. A project's .pi/mcp.json, read by the worker at each location, may add stdio servers that run there and switch the owner's servers on or off; the manager refuses everything else. A lease gets servers only on environments its session works on. --- apps/p2/docs/handoff.md | 10 +- apps/p2/docs/mcp-durable.md | 668 +++++++++++++++++ apps/p2/docs/p2-notes.md | 48 +- apps/p2/package.json | 2 + apps/p2/plugins/envs/manager.ts | 10 + apps/p2/plugins/mcp/config.ts | 189 +++++ apps/p2/plugins/mcp/contract.ts | 146 ++++ apps/p2/plugins/mcp/environments.ts | 44 ++ apps/p2/plugins/mcp/instances.ts | 346 +++++++++ apps/p2/plugins/mcp/leases.ts | 78 ++ apps/p2/plugins/mcp/package.json | 5 + apps/p2/plugins/mcp/placement.ts | 249 +++++++ apps/p2/plugins/mcp/project.ts | 94 +++ apps/p2/plugins/mcp/server.ts | 388 ++++++++++ apps/p2/plugins/mcp/tools.ts | 114 +++ apps/p2/plugins/mcp/transports.ts | 94 +++ apps/p2/plugins/mcp/windows-command.ts | 49 ++ apps/p2/plugins/mcp/worker.ts | 218 ++++++ apps/p2/plugins/models/fake.ts | 7 +- apps/p2/src/api/v1/index.ts | 1 + apps/p2/src/hosts/plugins.ts | 6 + apps/p2/src/runtime/chord.ts | 13 +- apps/p2/test/fixtures/mcp-server.mjs | 64 ++ apps/p2/test/p2/mcp-environments.test.ts | 55 ++ apps/p2/test/p2/mcp-instances.test.ts | 139 ++++ apps/p2/test/p2/mcp-leases.test.ts | 89 +++ apps/p2/test/p2/mcp-project.test.ts | 93 +++ apps/p2/test/p2/mcp-tools.test.ts | 104 +++ apps/p2/test/p2/mcp-windows-command.test.ts | 55 ++ apps/p2/test/p2/mcp-worker.test.ts | 108 +++ apps/p2/test/p2/mcp.test.ts | 764 ++++++++++++++++++++ apps/p2/test/p2/plugin-layout.test.ts | 3 + package-lock.json | 2 + 33 files changed, 4242 insertions(+), 13 deletions(-) create mode 100644 apps/p2/docs/mcp-durable.md create mode 100644 apps/p2/plugins/mcp/config.ts create mode 100644 apps/p2/plugins/mcp/contract.ts create mode 100644 apps/p2/plugins/mcp/environments.ts create mode 100644 apps/p2/plugins/mcp/instances.ts create mode 100644 apps/p2/plugins/mcp/leases.ts create mode 100644 apps/p2/plugins/mcp/package.json create mode 100644 apps/p2/plugins/mcp/placement.ts create mode 100644 apps/p2/plugins/mcp/project.ts create mode 100644 apps/p2/plugins/mcp/server.ts create mode 100644 apps/p2/plugins/mcp/tools.ts create mode 100644 apps/p2/plugins/mcp/transports.ts create mode 100644 apps/p2/plugins/mcp/windows-command.ts create mode 100644 apps/p2/plugins/mcp/worker.ts create mode 100644 apps/p2/test/fixtures/mcp-server.mjs create mode 100644 apps/p2/test/p2/mcp-environments.test.ts create mode 100644 apps/p2/test/p2/mcp-instances.test.ts create mode 100644 apps/p2/test/p2/mcp-leases.test.ts create mode 100644 apps/p2/test/p2/mcp-project.test.ts create mode 100644 apps/p2/test/p2/mcp-tools.test.ts create mode 100644 apps/p2/test/p2/mcp-windows-command.test.ts create mode 100644 apps/p2/test/p2/mcp-worker.test.ts create mode 100644 apps/p2/test/p2/mcp.test.ts diff --git a/apps/p2/docs/handoff.md b/apps/p2/docs/handoff.md index 6e56d55f7..fd86b18ff 100644 --- a/apps/p2/docs/handoff.md +++ b/apps/p2/docs/handoff.md @@ -65,7 +65,7 @@ npm test # full suite; TUI tests need Node 26, browser tests ``` Environment variables: `docs/p2-notes.md` (Running). Configuration in the data dir: `models.json`, `settings.json` -(incl. `codemode`, default on in p2), `envs.json`, `auth.json`, `radius.json`, `forwards.json`. +(incl. `codemode`, default on in p2), `envs.json`, `mcp.json`, `auth.json`, `radius.json`, `forwards.json`. **Sign-in.** Requests from the device itself are the owner. Other devices get owner rights with a link from `npm run owner-link` (single use, 10 minutes, long-lived signed cookie). Guests join through share links. Requests other @@ -96,7 +96,7 @@ verifying an agent's files on their own before committing. Node 24: `~/node-v24. - **Runtime** (`src/runtime`): shared contracts and helpers (principals, worker and Harness, Durable registry, extension switches, input entries, settings, logs, fetch). - **Plugins**: chat, chat-mode, sessions, subagents, agent (one-shot agents), codemode, coding-tools, prompt, - models, login, search, status, drafts, files, send-file, artifacts, workspace, file-index, terminals, envs, ports, + models, login, search, status, drafts, files, send-file, artifacts, workspace, file-index, terminals, envs, mcp, ports, radius, sharing, devices, proxy-marker, logs, dev-tools, powershell, web-shell. Before writing one, read Chord's tutorial (`packages/chord/docs/tutorial/`) and plugin guide (`packages/chord/docs/plugin-authoring.md`), then an existing p2 plugin of the same shape. @@ -109,7 +109,7 @@ verifying an agent's files on their own before committing. Node 24: `~/node-v24. - **Durable** (`packages/durable`) runs the agent: conversations, inputs as custom entries (`p2.input`, `chat.mention`), display-only writes placed immediately, code mode (QuickJS sandbox; scripts call tools as `tools.`, one-shot agents via `tools.agent`), compaction with visible failures, deferred tools and - `tool_search` (unused until MCP lands). + `tool_search` (MCP tools are deferred). ## The TUI @@ -155,7 +155,9 @@ From `AGENTS.md` and this project's history: 3. **Cleanup leftovers:** two HTTP helpers were not merged because their semantics differ (byte ranges: files serves `bytes=-` whole, workspace answers 416; body readers: sharing answers 400 past the limit, devices 413); decide one behavior each. -4. **MCP:** when it lands in Durable, mark MCP tools deferred and install `tool_search`. +4. **MCP:** `plugins/mcp` (WP6 and WP7 of `docs/mcp-durable.md`: owner `mcp.json`, host stdio and HTTP, deferred + tools, `tool_search`, servers on environments, project `.pi/mcp.json`); OAuth (WP8) and code mode namespaces (WP9) + are open. 5. **npm:** deprecate `@earendil-works/pi-client`, `pi-protocol` and `pi-server` (1.1.0), which no longer exist. 6. `docs/p2-notes.md` TODO: the TypeScript 5.9.3 pin (Termux), an isolating plugin loader, sandboxing third-party facets. diff --git a/apps/p2/docs/mcp-durable.md b/apps/p2/docs/mcp-durable.md new file mode 100644 index 000000000..5777d4766 --- /dev/null +++ b/apps/p2/docs/mcp-durable.md @@ -0,0 +1,668 @@ +# MCP for Durable and p2: work plan + +This file is the brief for agents implementing MCP (Model Context Protocol) support for p2, the Durable-based app in +`apps/p2`. Every work package (WP) below is self-contained: it says where the work happens, what to build, how to test +it, and when it is done. Read sections 1 to 4 before any work package, then only the package you are given. + +The human-readable plan with background lives at https://radius.earendil.com/artifact/01m6em2r8gercbx2rn2y8mwwpq +(revision 4). Where it and this file differ, this file wins. + +--- + +## 1. Repository and rules + +Everything is in `earendil-works/pi` (local checkout `~/Development/pi-mono`, branch `main`): the packages under +`packages/` (`env`, `mcp`, `durable`, `coding-agent`, `chord`, ...) and p2 at `apps/p2`. Until p2 moved there (commit +`05dc8d239`), it lived in the private repository `earendil-works/pi-private`, which is no longer used. + +Rules (the repository's `AGENTS.md` overrides anything here): + +- **Packages and p2 change together.** A fix to `packages/env` or `packages/mcp` found during p2 work is part of the + same tree; it still gets its own commit and changelog entry. +- Several agents may work in the same checkout. Stage explicit paths only; never `git add -A`, `git stash`, + `git reset --hard` or `git checkout .`. +- After code changes: `npm run check` from the repository root (formatting, lint, types), and after p2 changes also + `cd apps/p2 && npm run check` (p2 is outside the root check). It must be clean except for + errors you can show are unrelated and pre-existing. If it fails on missing modules or stale generated model data, run + `npm install --ignore-scripts` and `npm run hydrate:model-data` first (this does not change tracked files). +- Tests: run the package's own tests, never the whole suite. From the package directory: + `node "$(git rev-parse --show-toplevel)/node_modules/vitest/dist/cli.js" --run test/.test.ts`. + p2's tests use `node --test` (see `apps/p2/package.json`). +- The pi-env daemon is Rust: `cd packages/env/daemon && cargo fmt --check && cargo clippy --locked --all-targets -- -D + warnings && cargo test --locked`. Also lint for Windows: `cargo clippy --locked --all-targets --target + x86_64-pc-windows-msvc -- -D warnings`. Rebuild the debug daemon (`npm run build:daemon` in `packages/env`) before + running pi-env's TypeScript tests. +- Public packages get a `CHANGELOG.md` entry under `## [Unreleased]` (`### Added`, `### Changed`, `### Fixed`). +- Commit only when asked, one commit per work package, message `feat(): ...` / `fix(): ...`. +- No `any`, no inline imports, erasable TypeScript only, no runtime checks of values TypeScript already guarantees; + validate only at trust boundaries (wire input, configuration files, data from another process). + +--- + +## 2. What we are building + +p2 gets MCP servers. The design, in short: + +- **One manager on the p2 server owns every MCP connection and process.** Session workers never spawn an MCP server or + hold a credential. They talk to the manager through one Chord object per worker principal. +- **Placement.** A stdio server runs on the p2 host or on an execution environment from p2's `envs.json` (an SSH + machine with the pi-env daemon). On an environment, the manager starts it through the daemon's `process` operation + (WP1, done). +- **Instances.** Stdio servers run once per session and location by default; `"shared": true` in the owner's + configuration makes one process for everybody. Streamable HTTP servers are shared by default. +- **Configuration.** The owner's `/mcp.json` (trusted) plus a project's `/.pi/mcp.json`, which the + worker reads at its location and sends to the manager (untrusted, restricted, see WP7). +- **Tools.** Workers turn the servers' tools into Durable tools named `mcp____`, deferred (found through + `tool_search`) or callable only from code mode. +- **OAuth.** Only HTTP servers use MCP OAuth. Sign-in links point at p2 (`/mcp/sign-in/`); when one is opened, p2 + picks the redirect from the request's principal: loopback (`http://127.0.0.1:/callback`) for a browser on the + p2 host, p2's `/mcp/callback` on the Radius or public origin otherwise, and pasting the final redirect URL always works. + +```text + worker host (session s1) server host + ┌──────────────────────────────────┐ ┌──────────────────────────────────────────────────────┐ + │ MCP worker object │ hold as │ p2.mcp: remote view, one per principal │ + │ · reads .pi/mcp.json per ├─────────►│ worker variant ──┐ │ + │ location via its ExecutionEnv │ worker │ ▼ │ + │ · contributes Durable tools │ {s1} │ p2.mcp.manager: local, host lifetime │ + └──────────────────────────────────┘ │ instances, leases, OAuth, pi-env connections │ + │ owner variant ◄── web panel, TUI │ + └───────┬──────────────────┬──────────────────┬────────┘ + stdio on the p2 host stdio on env "gpu" Streamable HTTP +``` + +--- + +## 3. Status and order + +| WP | Where | What | State | Depends on | +| --- | --- | --- | --- | --- | +| 1 | `packages/env` | daemon `process` operation, `openProcess()` | **done**: `9c9ce8547`, fix `032656b6c` | none | +| 2 | `packages/mcp` | transport over a byte pipe | **done**: `c158fa2ee` | none | +| 3 | `packages/mcp`, `packages/coding-agent` | OAuth sign-in split into start/complete, moved out of coding-agent | **done**: `bb266bf35`, fix `797e122cb` | none | +| 4 | `packages/mcp`, `packages/coding-agent` | shared naming and config helpers (optional) | **done**: `76c040a2a` | 3 | +| 5 | p2 | **gate:** p2's Chord v2 port has landed; WP1-3 present | **done**: p2 imported at `05dc8d239`, deps in "feat(p2): MCP servers on the host, on environments and from projects" | 1-3, the port | +| 6 | `apps/p2/plugins/mcp` | manager, view, worker object; owner config; host stdio and HTTP | **done**: "feat(p2): MCP servers on the host, on environments and from projects" | 2, 5 | +| 7 | `apps/p2/plugins/mcp` | environment placement; project `mcp.json` | **done**: "feat(p2): MCP servers on the host, on environments and from projects" | 1, 6 | +| 8 | `apps/p2/plugins/mcp` | OAuth: flows, routes, loopback, pasting; web and TUI | open | 3, 6 | +| 9 | `apps/p2/plugins/mcp`, `plugins/codemode` | code mode namespaces, progress, first-turn admission | open | 6 | + +Section 4.5 lists where the built APIs of WP2-4 differ from sections 5-7, section 4.6 where WP6-7 differ from sections +9-10. + +--- + +## 4. Facts every package relies on + +### 4.1 WP1 (done): the daemon's `process` operation + +In `packages/env` (commit `9c9ce8547`): + +- Daemon: `packages/env/daemon/src/process.rs`; protocol in `packages/env/docs/protocol.md` (section on `process`). +- Client: `packages/env/src/process.ts`, exported from `@earendil-works/pi-env`: + +```ts +export function openProcess(connection: Connection, options: ProcessOptions): Promise; +export interface ProcessOptions { + cwd: string; + argv: readonly string[]; // run directly, no shell + env?: Record; // added to the daemon's environment + inheritEnv?: boolean; // false: env is the whole environment + onStdout: (bytes: Uint8Array) => void; + onStderr?: (bytes: Uint8Array) => void; +} +export class RemoteProcess { + readonly pid: number; + readonly exited: Promise; // exit code; rejects (code "aborted") after close(), or on a lost connection + write(data: Uint8Array | string): void; // stdin, in order; dropped once stdin is closed or the program ended + closeStdin(): void; // after the input written before it + close(): void; // kills the process group +} +``` + +- A daemon released before WP1 fails `openProcess` with `RemoteError` code `not_supported` + (`hello.features` lacks `"process"`). +- A lost connection kills the program (the daemon kills everything it started). +- When the program exits on its own, the daemon kills what it left running in its process group (Unix) or its job + object (Windows) before reporting the exit (`7367bc823`). Known gap on Windows: a child started before the daemon + assigns the program to its job can escape (documented in `packages/env/docs/protocol.md`). +- **Windows:** like `exec` argv commands, batch files (`npx.cmd`, `uvx.cmd`) are refused; callers must start them + through `cmd.exe` (WP7). + +### 4.2 Durable pieces (public, `packages/durable`) + +- `ToolRegistration` (`packages/durable/src/harness/types.ts`): `name`, `description`, `parameters` (JSON Schema object), + `deferred?: true`, `callers?: ("model" | "tools")[]`, `replay?: "safe" | "unsafe"` (default unsafe), + `structuredOutputSchema?`, `outputLimits?`, `execute(args, api, context)` returning `ToolExecutionResult` + (`output`, `structuredOutput`, `isError`, `details`). `api.conversationId` and `api.env` identify the calling + conversation and its execution environment. +- `createToolSearchExtension()` from `@earendil-works/pi-durable/tool-search` + (`packages/durable/src/tool-search/tool-search.ts`): the `tool_search` tool that loads deferred tools. +- `createCodemodeExtension({ sandbox, namespaces })` from `@earendil-works/pi-durable/codemode`; `namespaces` are + `{ name, description?, instructions?, prefix }` (`packages/durable/src/codemode/describe.ts`). +- A tool call whose tool is not registered right now waits until it is (`packages/durable/src/harness/agent.ts`, + `addTools`), so a reconnecting MCP server does not fail a replayed call. + +### 4.3 pi-mcp today (public, `packages/mcp`) + +- `McpClient` (`src/client.ts`), transports in `src/transports/` (`stdio.ts`, `streamable-http.ts`, `in-memory.ts`, + the `McpTransport` interface and `TransportEvents` in `transport.ts`), `toLlmContent()` (`src/protocol/content.ts`). +- OAuth in `src/oauth/`: `McpOAuthProvider` and `McpOAuthStateStore` (`provider.ts`; the state persists + `codeVerifier` and `oauthState`, so a sign-in can complete in another request with the same store), + `authorizeMcp` (`flow.ts`), `OAuthCallbackServer` (`callback.ts`), discovery and errors. +- No dependency on other pi packages, and only `cross-spawn` at runtime. Keep it that way. + +### 4.4 p2 today and after the port + +- p2 runs on Chord v2; Chord v1 is deleted. +- Chord v2 design: `packages/chord/docs/design.md`. The parts MCP uses: `local` objects with empty params (host + lifetime, §5, §6), shared store with per-principal `remote` views (§6 "Pattern"), natives `self.open` (§13), routines + `self.on`/`self.onLive` (§14), changing registry contributions with `self.writable` (§17; the design's own example is + MCP tools), flows kept by their owner (§20), `ctx.incarnation` (§4), Chord calls from Durable code (§24). +- p2 code style for Chord v2 code: `AGENTS.md`, section "Chord v2 Code Style". Use the names the port introduces + (harness extension registry, host addresses, principals, HTTP routing, login flow); do not invent parallel ones. +- Existing p2 precedents to copy: `apps/p2/plugins/ports` (a local store plus a remote view with owner and worker + variants), `apps/p2/plugins/locations/server.ts` (worker-only methods fenced by the worker's generation), + `apps/p2/plugins/terminals/main.ts` (the server opening its own pi-env connections per environment), + `apps/p2/plugins/envs` (environment configuration, SSH options, host keys), `apps/p2/plugins/login` (provider OAuth + with `remote: principal.access !== "local"`), `apps/p2/plugins/radius/relay-access.ts` (requests relayed through + Radius), `apps/p2/plugins/powershell/worker.ts` (a deferred tool offered per conversation), `apps/p2/src/runtime/principals.ts` + (`OwnerPrincipal.access` is `"local" | "relay" | "device"`). After the port these files move or change; find their + v2 equivalents. + +### 4.5 As built (WP2-4): differences from sections 5-7 + +- **WP2:** `PipeTransport`, `McpPipe`, `McpPipeHandlers`, `PipeTransportOptions` from `@earendil-works/pi-mcp`; + pi-env's `RemoteProcess` is an `McpPipe` without an adapter. Shared framing in `src/transports/lines.ts`. +- **WP3:** `@earendil-works/pi-mcp/oauth` exports `startMcpSignIn`, `completeMcpSignIn`, `listenForLoopbackSignIn`, + `waitForLoopbackSignIn`, `responseFromRedirectUrl`, `createMcpAuthProvider`, `McpSignInCancelledError`; files + `sign-in.ts`, `loopback.ts`, `auth-provider.ts`, `client-settings.ts`. Differences: + - `store` is an `McpOAuthServerStore` (with `withRefreshLock`); start and the auth provider take the lock, never + call them while holding it. + - `startMcpSignIn` returns `{ kind: "authorized" }` or a plain-data `McpSignInRedirect` (`authorizationUrl`, + `redirectUrl`, `state`, `scope?`, all strings): store it with the flow. + - `completeMcpSignIn({ ...sameOptionsAsStart, authorization, response })`: pass start's `redirectUrl` (not + `authorization.redirectUrl`), the kept start result, and a response with `state` (checked). A sign-in replaced by + a newer one is refused. + - `clientMetadataDocuments: { baseUrl, listsRedirectUrl(url) }`; without it, dynamic registration only (p2 remote). + - Start refreshes with the stored client first, whatever the redirect URI; `authorizeMcp` has `refreshOnly`, so a + failed refresh never disturbs a pending sign-in (`3ffb210d9`: the refresh uses the app's `redirectUrl` when the + documents list it; pass `redirectUrl` to `createMcpAuthProvider` too). + - The store's `withRefreshLock` throws `McpRefreshLockBusyError` when the lock stays held (start then goes to the + browser step); other lock errors fail the sign-in. Aborting ends the wait for the lock. +- **WP4:** from `@earendil-works/pi-mcp`: `createMcpToolName`, `mcpNamespace`, `normalizeMcpInputSchema`, + `limitMcpContent(content, saveOutput?)`, `MCP_OUTPUT_MAX_BYTES`, `validateMcpServerConfig(name, raw, options?)` + returning `{ kind: "stdio" | "http", config }` (unknown fields kept as `unknown`) or `{ kind: "invalid", error }`, + `isMcpServerOverride`. Apps validate their own fields (p2: `shared`, `runOn`, project rules) on the result. + `"cimd"` is refused unless `checkMetadataDocumentCallback` is passed (p2 should not pass it). + +### 4.6 As built (WP6-7): differences from sections 9-10 + +- **Leases, not load/unload:** the worker variant has `setLocations({ locations, projects? })` (every location the + session's conversations work at; leases elsewhere end) and `call({ location, server, tool, args })`; the manager has + `setLocations` and `releaseLeases` with the caller's session, generation and incarnation. Worker state: `servers` + and `errors` (the session's leases' project refusals). +- **Project files as text:** the worker sends `/.pi/mcp.json` as read (`{ location, text }`, at most 256 KiB, or + `{ location, error }` when unreadable; absent: none); the manager parses and checks it (`project.ts`) at each + placement. A project override may also turn on an owner server that is `"enabled": false`. +- **Where sessions work:** a lease gets servers only on an environment its session works on as the server records it + (the catalog's session, the subagent store's own locations). This fences stale or confused workers to their machines, + not a compromised worker (see `McpManager`'s doc comment). +- **Environments:** connections through the envs plugin's SSH manager (`EnvManager.connection`), one per environment + and revision; an instance on an environment that is its location's machine gets the location's cwd as its base and + root, otherwise the remote home. Windows: the `cmd.exe` command line goes in `P2_MCP_COMMAND` (the daemon quotes argv + itself and has no verbatim mode). +- **Not watched:** a changed project `mcp.json` is read again at the worker's next `setLocations` (a commit, a move, + the view re-activating), not when the file changes. + +--- + +## 5. WP2: pi-mcp transport over a byte pipe + +**Where:** `packages/mcp`. + +**Why:** p2 runs stdio MCP servers on remote machines through `openProcess()` (WP1). pi-mcp must not depend on pi-env, +so it gets a transport over an abstract pipe that `RemoteProcess` satisfies without an adapter class. + +**Build:** + +1. `src/transports/lines.ts` (new): the newline-delimited JSON-RPC framing now inside `StdioTransport.handleStdout` + (`src/transports/stdio.ts`): accumulate bytes, split at `\n`, strip a trailing `\r`, skip blank lines, enforce + `maxMessageBytes` (default `DEFAULT_MAX_MESSAGE_BYTES`), parse with `parseJsonRpcMessage`, report errors. Plus the + bounded stderr tail (`maxStderrBytes`, default 64 KiB). One class, used by both transports. +2. `src/transports/stdio.ts`: use `lines.ts`. Behavior must not change; `test/stdio.test.ts` must pass unmodified. +3. `src/transports/pipe.ts` (new), exported from `src/index.ts`: + + ```ts + /** A running program's stdio, as `@earendil-works/pi-env`'s `RemoteProcess` provides it. */ + export interface McpPipe { + write(data: Uint8Array | string): void; + closeStdin(): void; + /** Kill the program (and what it started). */ + close(): void; + /** Settles when the program has ended, whatever the reason. */ + readonly exited: Promise; + } + export interface PipeTransportOptions { + /** Start the program; `onStdout`/`onStderr` receive its output from then on. */ + open(handlers: { onStdout(bytes: Uint8Array): void; onStderr(bytes: Uint8Array): void }): Promise; + maxMessageBytes?: number; + maxStderrBytes?: number; + onStderr?: (chunk: string) => void; + /** After closing stdin, how long the program may take to exit before `close()` kills it. Default 2000. */ + closeTimeoutMs?: number; + } + export class PipeTransport extends TransportEvents implements McpTransport { ... } + ``` + + - `start()` calls `open`; a rejection rejects `start()`. Once `exited` settles, an incomplete trailing message is an + error (as in `StdioTransport`), then `emitClose()`. + - `send()` writes `JSON.stringify(message) + "\n"`; after `close()` or the program's end it throws + `McpConnectionClosedError`. + - `close()`: MCP's stdio shutdown: `closeStdin()`, wait for `exited` up to `closeTimeoutMs`, then `close()`. + - `stderr` getter with the tail, like `StdioTransport`. +4. README (`packages/mcp/README.md`): a short section with the pi-env example: + + ```ts + const transport = new PipeTransport({ + open: (handlers) => openProcess(connection, { cwd, argv, ...handlers }), + }); + ``` + +5. `CHANGELOG.md`: `### Added` entry. + +**Tests:** `test/pipe.test.ts` (new): + +- with an in-memory fake `McpPipe`: framing (split chunks, several messages in one chunk, `\r\n`, blank lines), the size + limit, an incomplete message at exit, `send` after close, the shutdown order (stdin closed first; `close()` only after + the timeout; not at all when the program exits in time), `start` rejecting when `open` rejects; +- end to end: a `McpClient` over `PipeTransport` whose `open` spawns `test/fixtures/stdio-server.mjs` with + `node:child_process` (a 20-line adapter in the test) completes `initialize`, `tools/list` and `tools/call`. + +**Done when:** tests pass, `test/stdio.test.ts` unchanged and green, `npm run check` clean, README and changelog +updated. pi-mcp still has no dependency on pi-env. + +--- + +## 6. WP3: OAuth sign-in split into start and complete, moved into pi-mcp + +**Where:** `packages/mcp/src/oauth` (new code) and +`packages/coding-agent/src/extensions/mcp` (callers). + +**Why:** p2 must run the same sign-in as `pi mcp login`, but the authorization code reaches p2 later and in another +request (its `/mcp/callback` route, a pasted URL, or a loopback server it runs). Today +`signInMcpServer` in `packages/coding-agent/src/extensions/mcp/oauth.ts` opens its own loopback server and waits. + +**Read first:** `packages/coding-agent/src/extensions/mcp/oauth.ts` in full, `packages/coding-agent/src/core/mcp-servers.ts` +(`McpOAuthConfig`, `isLoopbackRedirectUri`, `isClientMetadataDocumentCallback`), its callers +`packages/coding-agent/src/extensions/mcp/index.ts` (`signIn`, around `runtime.signInMcpServer`) and `runtime.ts`, and +`packages/mcp/src/oauth/*`. + +**Move into `packages/mcp/src/oauth/sign-in.ts`** (new; export from `src/oauth/index.ts`): + +| Today in coding-agent `oauth.ts` | In pi-mcp | Change | +| --- | --- | --- | +| `McpOAuthSettings`, `callbackSettings`, `mergeScopes`, `timedFetch` | same names | none | +| `offersClientMetadataDocument`, `clientMetadataDocument`, `callbackId` | same | the document base URL (today `https://pi.dev/oauth`) and the application name (today `APP_NAME`) become options | +| `responseFromRedirectUrl` | exported | none | +| `createMcpAuthProvider` | exported | `McpOAuthServerStore.withRefreshLock` stays an interface; pi-mcp gets no lockfile dependency | +| `signInMcpServer` | split | see below | + +The split: + +```ts +/** First half: refresh if possible, else prepare the browser step. */ +export async function startMcpSignIn(options: { + serverUrl: string; + store: McpOAuthStateStore; + settings: McpOAuthSettings; + redirectUrl: string; // chosen by the caller: loopback, or an https callback route + clientMetadataDocuments?: { baseUrl: string }; // omitted: dynamic registration only + clientName: string; + challenge?: OAuthChallenge; + signal?: AbortSignal; +}): Promise<{ kind: "authorized" } | { kind: "redirect"; authorizationUrl: URL; redirectUrl: URL; state: string }>; + +/** Second half: exchange the code. `response` from a callback server, a route, or `responseFromRedirectUrl`. */ +export async function completeMcpSignIn(options: { + serverUrl: string; + store: McpOAuthStateStore; + settings: McpOAuthSettings; + redirectUrl: string; // the one passed to start (the flow may have made it server-specific) + response: { code: string; iss?: string }; + clientMetadataDocuments?: { baseUrl: string }; + clientName: string; + signal?: AbortSignal; +}): Promise; +``` + +- Keep every rule of today's function: a fresh `state` per sign-in; keep or drop the registered client depending on its + redirect URI and `clientRegistration`; step-up scope (`stepUpScope`, `mergeScopes`); the server-specific redirect + path for Client ID Metadata Documents (`/callback/`); timeouts through `timedFetch`; cancellation via + `signal` reported as `McpSignInCancelledError` (move that class too). +- `OAuthCallbackServer` stays in pi-mcp; the loopback-listening part of today's function (`listenForCallback`, + `waitForAuthorizationResponse`) becomes a small helper in pi-mcp, `waitForLoopbackSignIn(...)`, or stays in + coding-agent if only it uses it. Decide by whether p2 needs it: p2 does (loopback for local browsers), so move it. +- coding-agent keeps: `McpOAuthCredentialStore` (`mcp-auth.json`, `proper-lockfile`, `AuthStorageBackend`, legacy + keys), the callback page HTML, `pi mcp login`/`/mcp` UI. `signInMcpServer` in coding-agent becomes a thin wrapper: + start, show the URL, wait for loopback or pasted URL, complete. + +**Tests:** + +- pi-mcp: `test/sign-in.test.ts` (new) with an authorization server built on `test/helpers.ts` (`listen`, `readBody`, + `closeServers`) the way `test/oauth.test.ts` builds its servers, or on coding-agent's + `packages/coding-agent/test/suite/mcp-oauth-server.ts` moved into pi-mcp's tests: start then complete in two calls with one store; complete with a + pasted URL; refresh path returns `authorized`; metadata document chosen only when the server supports it and the + redirect is one the documents list; a mismatched `state` or redirect path is rejected; cancellation. +- coding-agent: unchanged and green: `test/mcp-oauth-refresh.test.ts`, `test/suite/agent-session-mcp-oauth.test.ts`, + `test/mcp-command.test.ts`, `test/mcp-extension.test.ts`. + +**Done when:** pi-mcp exports the functions above; coding-agent's behavior is unchanged (its tests pass without +edits beyond imports); both changelogs updated (`packages/mcp/CHANGELOG.md` Added, `packages/coding-agent/CHANGELOG.md` +nothing user-visible, so no entry unless behavior changed). + +--- + +## 7. WP4 (optional): shared naming and config helpers + +**Where:** `packages/mcp` (new `src/naming.ts`, `src/config.ts`), callers in +`packages/coding-agent/src/core/mcp-servers.ts` and `src/extensions/mcp/tools.ts`. + +**Why:** p2 must name tools exactly like pi, so `tool_search` ranking and code mode namespaces behave the same. + +**Build:** + +- `createMcpToolName(server, tool, isTaken)` and `mcpNamespace(server)` (today in coding-agent `extensions/mcp/tools.ts` + and `core/mcp-servers.ts`), the input-schema normalization (object type, `properties` present) and the result size + limit (`MCP_OUTPUT_MAX_BYTES`, `limitMcpContent`) without pi-tui rendering. +- Parsing and validation of the standard `mcpServers` shape: stdio (`command`, `args`, `env`, `cwd`), HTTP (`url`, + `headers`), `oauth`, `enabled`. pi-only fields (`exposure`, `toolExposure`, `auth.provider`, `autoEnableCodemode`) stay + in coding-agent, which extends the shared parser. + +**Done when:** coding-agent imports these from pi-mcp with no behavior change (its MCP tests pass); pi-mcp tests cover +the moved functions. + +If skipped, WP6 copies `createMcpToolName` and `mcpNamespace` into p2 and notes the duplication. + +--- + +## 8. WP5 (gate): p2 on Chord v2, packages present + +Not MCP work, done. Before WP6: + +1. p2 imports Chord v2 (`rg -l "@earendil-works/chord/old" apps/p2` finds nothing). +2. WP1-3 (and 4) are in the tree p2 builds against. +3. Add `"@earendil-works/pi-mcp": ""` and `"@earendil-works/pi-env"` (if p2 does not have it yet) to + `apps/p2/package.json`; refresh the lockfile with `npm install --package-lock-only --ignore-scripts`; commit lockfile + changes with `PI_ALLOW_LOCKFILE_CHANGE=1`. + +--- + +## 9. WP6: manager, view and worker object (host stdio and HTTP) + +**Where:** `apps/p2/plugins/mcp/` (new plugin; register it where the port registers built-in +plugins). Tests in `apps/p2/test/p2/mcp*.test.ts`. + +**Files** (one concept per file, per "Chord v2 Code Style"): + +| File | Content | +| --- | --- | +| `contract.ts` | the `p2.mcp.manager` local contract, the `p2.mcp` remote contract, schemas (`Location`, `Instance`, `Tool`, `ProjectConfig`, `CallResult`, `SignInFlow`) | +| `config.ts` | reading and validating `/mcp.json` (trust boundary), following changes | +| `instances.ts` | the supervisor native: instances keyed per section 9.2, leases, connect, reconnect with backoff, tool-list changes | +| `server.ts` | the facet: serves the manager and the view | +| `worker.ts` | the worker object: holds the view, loads locations, contributes the Durable extension | +| `tools.ts` | MCP tool → Durable `ToolRegistration` | + +### 9.1 Contracts + +```ts +const Location = obj({ env: union([str(), nul()]), cwd: str() }); // p2's location; env null: the p2 host +const Instance = obj({ + id: str(), + server: str(), + source: oneOf("owner", "project"), + location: union([Location, nul()]), // null: shared, or HTTP + runsOn: union([literal("host"), obj({ env: str() })]), + status: oneOf("starting", "connected", "sign-in", "failed"), + error: opt(str()), + tools: arr(Tool), // name, title, description, inputSchema, annotations +}); + +export const McpManager = local("p2.mcp.manager", { + version: 1, + params: {}, + members: { + instances: arr(Instance), + load: method(obj({ session: str(), location: Location, project: union([ProjectConfig, nul()]) }), Loaded), + unload: method(obj({ session: str(), location: Location })), + release: method(obj({ session: str() })), + call: method(obj({ session: str(), instance: str(), tool: str(), args: JsonObject }), CallResult), + restart: method(obj({ instance: str() })), + // sign-in members: WP8 + }, +}); + +export const Mcp = remote("p2.mcp", { + version: 1, + hostAddress: /* the server, as the port addresses it */, + params: {}, + variants: { + worker: { + state: { servers: arr(Instance) }, + methods: { load, unload, call }, // session filled in by the view from the principal + }, + owner: { + state: { instances: arr(Instance) }, + methods: { restart }, + }, + }, +}); +``` + +- The view's factory returns the worker variant for a worker principal, the owner variant for an owner. Guests and + other principals get no access: authority checks live in the view's methods (design §7). +- Worker calls carry the caller's incarnation (`ctx.incarnation`); the manager records the incarnation that loaded each + lease and refuses calls from an older one. +- `servers` is a derive over the manager's `instances` filtered to the principal's session plus the shared instances + the session uses. + +### 9.2 Instances + +| Kind | Default | Key | Notes | +| --- | --- | --- | --- | +| stdio, `shared` unset | per session | `(session, location, server, config hash)` | roots: the location's cwd | +| stdio, `"shared": true` | opt-in, owner config only | `(server, runsOn, config hash)` | configured roots; state shared by all sessions | +| Streamable HTTP | shared | `(server, config hash)` | `"shared": false`: one per session, still in the server | + +- The configuration hash covers command, args, env, cwd and URL. +- **Leases:** each view holds leases on the instances it loaded. `unload` releases one location; when the view ends + (its worker's reference went with the link, after `linger`), a native of the view releases all of them through + `release`. Unshared instances close with their last lease; shared instances stay while any lease or owner entry needs + them. +- **Reconnect:** a lost connection reconnects with exponential backoff (1 s doubling to 60 s); tools stay listed while + reconnecting (status `starting`), calls fail with a clear error. A tool-list-changed notification refreshes `tools`. +- **Shutdown:** closing an instance closes its client; stdio follows MCP's shutdown (stdin closed, then kill), which + `StdioTransport` and `PipeTransport` implement. + +### 9.3 Host stdio and HTTP in this package + +- Host stdio: `StdioTransport` from pi-mcp (`runOn: "host"`, and `"location"` for local sessions). +- HTTP: `StreamableHttpTransport` with headers from the owner config (`${VAR}` from the p2 host's environment, owner + config only). OAuth arrives in WP8; until then an HTTP server that answers 401 shows status `sign-in`. +- `McpClient` options: `name: "p2"`, version from p2's package, `roots` from the instance's location cwd for unshared + instances (`file://` URL), none for shared ones. + +### 9.4 Worker side + +- A local, host-lifetime object in the worker holds `Mcp` (the server sees it as `worker{session}`). +- At start and after every relocation of the session, and for every subagent with its own location: read + `/.pi/mcp.json` through that location's `ExecutionEnv` (`readTextFile`; absent file: `project: null`), call + `load`. Call `unload` for locations nothing works at any more. +- Contribute one Durable `Extension` through the harness extension registry as a `self.writable`; `self.on(view.servers, + ...)` rebuilds it. Tools: + - name via `createMcpToolName` (WP4, else copied), description from the MCP tool, `parameters` normalized to an object + schema; + - `deferred: true` (found through `tool_search`); + - `execute`: locate the calling conversation (`api.conversationId`, through the harness's environments), pick the + instance for that location, call `view.call` from Durable code (design §24's helper), convert the result with + `toLlmContent()`, pass `structuredContent` through as `structuredOutput` only if the tool declares an output schema, + `isError` from the MCP result; replay stays `unsafe`. +- Register `createToolSearchExtension()` (nothing installs `tool_search` in p2 today). + +### 9.5 Owner configuration (`/mcp.json`) + +```json +{ + "mcpServers": { + "filesystem": { "command": "npx", "args": ["-y", "@modelcontextprotocol/server-filesystem", "."] }, + "browser": { "command": "npx", "args": ["-y", "@playwright/mcp"], "shared": true, "runOn": "host" }, + "tools-on-gpu": { "command": "/opt/tools/mcp", "runOn": { "env": "gpu" } }, + "docs": { "url": "https://example.com/mcp", "headers": { "Authorization": "Bearer ${DOCS_TOKEN}" } } + } +} +``` + +`runOn`: `"location"` (default for unshared stdio: where the conversation works), `"host"` (default for shared), or +`{ "env": "" }`. Until WP7, `"location"` on a remote session and `{ env }` show status `failed` with +"environments arrive in WP7". + +**Tests** (`apps/p2/test/p2/mcp.test.ts`, `node --test`), with `test/fixtures/stdio-server.mjs`-style fixture servers +copied or referenced from pi-mcp: + +- owner config loads; invalid entries are reported and skipped; +- a session loads; its worker gets the tools; a call returns the result; `tool_search` finds the tool; +- two sessions get two processes for an unshared server and one for a shared server; +- unload and view end close unshared instances; shared ones survive; +- an older worker incarnation's call is refused; +- a crashed server reconnects with backoff and its tools come back; +- a guest gets nothing. + +**Done when:** the tests pass, `npm run check` clean, and a manual run (throwaway session under `/tmp`, separate data +dir and port, per "No hand tests on real sessions" in `apps/p2/docs/handoff.md`) shows an MCP tool used by the agent. + +--- + +## 10. WP7: environment placement and project configuration + +**Where:** `apps/p2/plugins/mcp/` (`instances.ts`, new `placement.ts`, `config.ts`). + +### 10.1 Placement on environments + +- For `runOn: "location"` on a remote session and for `{ env }`, the manager opens one pi-env `Connection` per + environment, built from the envs plugin's configuration, SSH options and known hosts, as + `apps/p2/plugins/terminals/main.ts` does with `sshConnection(...)`. One connection per environment, shared by all + instances there; closed when no instance uses it. +- Start the server with `openProcess(connection, { cwd, argv, env, onStdout, onStderr })` behind pi-mcp's + `PipeTransport` (WP2). +- `openProcess` failing with `not_supported`: status `failed`, error "the pi-env daemon on is too old; update it" + (pi-env deploys its packaged daemon by hash, so a p2 update usually fixes it). +- **Windows environments** (`nodePlatform(await connection.info()).platform === "win32"`): resolve the program; if it is + not an `.exe`/`.com` (for example `npx`, which is `npx.cmd`), run it as `["cmd.exe", "/d", "/s", "/c", ]`. Quote like `cross-spawn` does for cmd (escape `^&|<>()%!"` meta characters). Unit-test the quoting. +- `runOn: "location"` on a local session uses the host path (WP6). + +### 10.2 Project configuration + +The worker sends `/.pi/mcp.json` with `load`; the manager validates it at that boundary. Project entries are +untrusted (a repository wrote them, and the agent can edit them). Rules: + +| Rule | Why | +| --- | --- | +| stdio runs at its location only; `runOn` and `shared` are refused | the agent can already run commands there; anywhere else a repository would gain code execution on the p2 host or another machine | +| no `${VAR}` from the p2 host's environment | a project could otherwise put the host's secrets in a header or argument; variables in `env` resolve at the location | +| no `auth.provider`, no reuse of another server's OAuth client | as in pi: a repository does not choose where a credential goes | +| an entry without `command`/`url`/`type` only overrides `enabled` of the owner entry with the same name | as in pi (`packages/coding-agent/src/extensions/mcp/config.ts`) | +| HTTP entries: **refused** in this WP | open decision (section 14); the p2 host would make the requests, so a project could reach addresses only the host can see | + +**Tests:** placement on a local "environment" through the local daemon (`localDaemon()`, as pi-env's tests do) so no +SSH is needed: a server on an environment starts, its tools appear, calls work, the connection is shared by two +instances, closing the last instance closes the connection; project rules each refuse what they should; Windows +quoting unit tests. + +--- + +## 11. WP8: OAuth flows, routes, loopback, pasting; web and TUI + +**Where:** `apps/p2/plugins/mcp/` (new `sign-in.ts`, routes in `server.ts`, `web.tsx`, +`tui.ts`). The web UI is p2 code again (`apps/p2/plugins/*/web.tsx`, `apps/p2/src/web`), ported with p2. + +**Credential store:** `/mcp-auth.json`, private (0600), keyed by server name and URL, with a refresh lock +(`proper-lockfile`, already a p2 dependency) implementing pi-mcp's `withRefreshLock`. Tokens never leave the server. + +**Flow** (a Chord flow kept by the manager, design §20: `start` → id; state with the current step; `answer`; `cancel`): + +1. An HTTP instance gets `McpOAuthAuthorizationRequiredError` → status `sign-in`. The tool error tells the model to ask + the user to sign in and gives the link from the worker's `signIn`. +2. `signIn({ instance })` (worker and owner variants) starts a flow and returns `{ id, link: "/mcp/sign-in/" }`. The + link is relative, so it opens on the origin the viewer uses. +3. Route `GET /mcp/sign-in/` (owners only): read the request's principal and choose the redirect: + + | Principal | Redirect URI | Code arrives through | Registration | + | --- | --- | --- | --- | + | `owner`, `access: "local"` | `http://127.0.0.1:/callback` (pi-mcp `OAuthCallbackServer` on the p2 host) | the loopback server | pi.dev Client ID Metadata Documents when supported, else dynamic registration | + | `owner`, `access: "relay"` | `https:///mcp/callback` (from `radius/relay-access.ts`) | route `GET /mcp/callback` | dynamic registration | + | `owner`, `access: "device"` on an allowed public host | `https:///mcp/callback` | route `GET /mcp/callback` | dynamic registration | + + Then `startMcpSignIn` (WP3) and answer 302 to the authorization URL. `authorized` (a refresh worked): redirect back + to p2 with a success page. +4. `GET /mcp/callback?code&state&iss` (owners only): find the flow by `state`, `completeMcpSignIn`, reconnect the + instance, show a success page. The flow is done; its state shows it. +5. Pasting: the owner variant's `answer({ id, step, redirectUrl })` uses `responseFromRedirectUrl` then completes. The UI + always offers it (an `ssh -L` port forward looks `local` but the browser is elsewhere). + +**UI:** + +- Web (`plugins/mcp/web.tsx`): an MCP panel next to Providers: servers with status and errors, tools count, restart, + sign in (opens the link), sign out, paste the redirect URL while a flow waits. +- TUI (`plugins/mcp/tui.ts`): the same as a command/overlay; opening the link uses the launcher's open-URL grant. +- Sign-in links in the transcript are ordinary relative links. + +**Tests:** a fake authorization server and MCP HTTP server (the one WP3's tests use): a 401 makes status `sign-in`; +the sign-in route as a local principal starts a loopback flow; as a relay/device principal it uses `/mcp/callback`; +the callback completes and the tools appear; a wrong `state` is refused; pasting completes; guests are refused on both +routes; tokens refresh without a browser. Verify by hand once through Radius that the cortex origin's cookie is sent on +the redirect back (section 14). + +--- + +## 12. WP9: code mode namespaces, progress, first-turn admission + +**Where:** `apps/p2/plugins/codemode/` (namespaces), `apps/p2/plugins/mcp/`. + +- **Namespaces:** one `CodemodeNamespace` per server (`name` = server, `prefix` = `mcpNamespace(server) + "__"`, + `description` from the server's `instructions` in `initialize`, if any) fed into `createCodemodeExtension` from the + `servers` state, through a registry the MCP plugin contributes to or a getter read at use (as codemode already does for + its sandbox). +- **Progress:** Chord calls return one result. Default: no MCP progress in the first version. If wanted: the manager + writes progress per call into a state document the worker follows and forwards through Durable's `api.output()` or + `details()`. +- **First turn:** decide whether p2's admission check waits for MCP servers still `starting` (default: wait up to 10 s + for instances of the session's location, then go on; deferred tools appear later through `tool_search`). + +--- + +## 13. Things that are easy to get wrong + +- A fix to `packages/env` or `packages/mcp` found during p2 work is its own commit, with its package's changelog entry. +- Do not run MCP processes in session workers. Everything that spawns, connects or holds a token is in the manager. +- Do not trust project `mcp.json`: validate at the manager, apply section 10.2. +- Tool names must match pi's (`createMcpToolName`), including the hash suffix for long names. +- Windows: `npx`/`uvx` are batch files; host stdio uses `cross-spawn` through `StdioTransport` (fine), environments need + the `cmd.exe` wrapping (WP7). +- The daemon kills a remote server when its connection drops; reconnect logic must expect that and restart it. +- Never print tokens, cookies or sign-in URLs with codes in logs. + +--- + +## 14. Open decisions (defaults to use until decided) + +| Decision | Default in this plan | +| --- | --- | +| Per session or per agent | per session and location; one-shot agents share their session's instance | +| Project HTTP servers | refused | +| Remote client registration | dynamic registration; per-brain metadata documents from Radius only if needed later | +| Host stdio transport | `StdioTransport` (cross-spawn); the local daemon only if a reason appears | +| MCP progress | not in the first version | +| First-turn admission | wait up to 10 s for the session's starting instances | +| Radius cookie on the redirect back | verify by hand in WP8; if it is not sent, relayed browsers fall back to pasting | diff --git a/apps/p2/docs/p2-notes.md b/apps/p2/docs/p2-notes.md index 36f7bc89a..67615fcec 100644 --- a/apps/p2/docs/p2-notes.md +++ b/apps/p2/docs/p2-notes.md @@ -27,9 +27,9 @@ P2_FAKE=1 npm start # http://127.0.0.1:8788, data in ~/.pi/p2 | Variable | Default | Meaning | | --- | --- | --- | -| `P2_DATA_DIR` | `~/.pi/p2` | data directory: `catalog.sqlite`, `sessions/.sqlite`, `settings.json`, `models.json`, `envs.json`, `auth.json`, `radius.json`, `forwards.json` | +| `P2_DATA_DIR` | `~/.pi/p2` | data directory: `catalog.sqlite`, `sessions/.sqlite`, `settings.json`, `models.json`, `envs.json`, `mcp.json`, `auth.json`, `radius.json`, `forwards.json` | | `PORT` | `8788` | listen port (loopback only), so p2 never collides with pi-mobile on 8787 | -| `P2_FAKE` | unset | `1`: the faux provider (`faux/faux-1`, echoes the prompt; `slow …` streams long; `rename-session ` calls the tool; `subagent`, `subagent-bad`, `subagent-answer`, `subagent-configure`, `subagent-send`, `subagent-archive`, `subagent-status` exercise subagents, `subagent <json>` calls the tool with those arguments, as do `team <json>`, `history_search|read|handoff <json>` and `location_envs|dirs|move <json>`; `codemode` runs a script calling bash and read, `codemode-restart` one whose slow middle call a restart interrupts, `codemode <script>` runs that script; `json <text>` answers exactly `<text>`; a one-shot agent whose task is not `json …`/`slow…` answers a small value matching its schema) | +| `P2_FAKE` | unset | `1`: the faux provider (`faux/faux-1`, echoes the prompt; `slow …` streams long; `rename-session <title>` calls the tool; `subagent`, `subagent-bad`, `subagent-answer`, `subagent-configure`, `subagent-send`, `subagent-archive`, `subagent-status` exercise subagents, `subagent <json>` calls the tool with those arguments, as do `team <json>`, `history_search|read|handoff <json>`, `location_envs|dirs|move <json>`, `tool_search <json>` and `mcp__<server>__<tool> <json>`; `codemode` runs a script calling bash and read, `codemode-restart` one whose slow middle call a restart interrupts, `codemode <script>` runs that script; `json <text>` answers exactly `<text>`; a one-shot agent whose task is not `json …`/`slow…` answers a small value matching its schema) | | `P2_IDLE_MS` | `300000` | idle time before a session worker stops | | `P2_PROVIDER`, `P2_MODEL` | unset | default model for new sessions after settings.json's `defaultModel` (both or neither; Pi Mobile's `PI_MOBILE_PROVIDER`/`PI_MOBILE_MODEL` work too) | | `P2_RADIUS` | unset | `0`: disable the outbound Radius brain connection even when signed in | @@ -972,6 +972,50 @@ translated phone drawer and intercept chat taps. reload during a script, SIGKILL and graceful restarts mid-call (finished calls do not rerun; the interrupted unsafe `bash` call returns an error the script catches); the removal mid-script (prompt error, turn ends, the script's thread goes); `codemode-plugin.test.ts` the sandbox across reload/removal. +- **MCP servers** (`plugins/mcp`, plan `docs/mcp-durable.md`, WP6 and WP7): the server's `McpManager` (local, + host lifetime) owns every MCP process and connection; workers never spawn one or hold a credential. `<P2_DATA_DIR>/mcp.json` + is the owner's `mcpServers` (pi-mcp's `validateMcpServerConfig`, plus p2's `shared` and `runOn`; invalid entries are + skipped and listed in `errors`, `${VAR}` in headers and env values from the p2 host's environment; watched: an edited + server restarts, a removed one closes; an invalid save keeps the last valid servers, also across a reload). Instances (`placement.ts`): stdio per session and location (key: session, + location, server, hash of command, args, env, cwd and timeout; roots the location's cwd), `"shared": true` once on + the host, HTTP shared unless `"shared": false` (one per session); a 401 shows `sign-in` (OAuth is WP8). Stdio + servers on an environment (`runOn: "location"` on a remote location, or `{ env }`) start through its pi-env daemon + (`openProcess` behind pi-mcp's `PipeTransport`, `transports.ts`): one connection per environment and revision of its + `envs.json` entry, shared by its instances and closed with the last (`environments.ts`), opened through the envs + plugin's SSH manager (`EnvManager.connection`); `~` is the remote home, relative paths are relative to the location's + cwd when it is on that machine (also the root), else to the home; on Windows everything but `.exe`/`.com` runs through + `cmd.exe` with cross-spawn's quoting, the line passed in a variable (`windows-command.ts`); a daemon without the + `process` operation fails for good with "update it"; an environment missing from `envs.json` shows `failed`. A lease + gets servers only on an environment its session works on as the server records it (the catalog's session, the + subagent store's own locations; else "not where this session works"). Projects: the worker reads `<cwd>/.pi/mcp.json` + at each location through that location's environment and sends the text with `setLocations` (256 KiB at most); the + manager checks it (`project.ts`): stdio servers only, run at that location for that session alone (no `runOn`, + `shared`, `auth`, `oauth`, no HTTP, `${VAR}` never expanded), an entry without `command`/`url`/`type` only sets + `enabled` of the owner's server of that name (so a disabled owner server can be turned on there), a redefinition of + an owner server is refused; refusals and unreadable files are the lease's `errors` (the worker's view, and the + owner's prefixed with the session). `Mcp` is the view: a worker + principal's `setLocations`/`call` (only the session's running current generation, as the location tools check; + session, generation and `ctx.incarnation` filled in by the view; `servers` without the owner's errors), the owner's + `instances`, `errors`, `restart`; guests get nothing. Lease rules are pure functions (`leases.ts`): a generation sets + exactly its locations (leaving one out ends that lease), a newer generation takes the session over and refuses an + older one (decided inside the leases' change), calls go through the setting incarnation's lease, and the view's end + (its worker's link went, after a 30 s linger, so a worker object taking it again keeps its processes) releases its + generation's leases; a reload keeps them and the last listed tools (in-memory + documents of the manager). Unshared instances close with their last lease, shared ones stay while configured. A lost + connection reconnects with backoff (`RECONNECT`: 1 s doubling to 60 s, reset after a minute up), tools still listed; + `notifications/tools/list_changed` refreshes them; call errors never carry a server's stderr (the owner's `error` + does). The worker object (`p2.mcp-worker`) sends `setLocations` with every location its conversations work at (the + session's, active subagents' own) at start, after moves and subagent commits, and whenever the view re-activates (a + refusal because a newer generation runs ends it), and + contributes one Durable extension `mcp` (a `self.writable`, rebuilt only when the tool list changes): deferred tools + named by `createMcpToolName` (every tool of a colliding name gets the hash suffix, as in pi), results through + `toLlmContent` and `limitMcpContent`, `structuredOutput` only for tools with an output schema. The worker facet also + installs `tool_search`. Tests: `mcp.test.ts` (manager and views on a bare host with `test/fixtures/mcp-server.mjs` and + an HTTP fixture; projects, where sessions work, and an environment through the debug pi-env daemon, skipped without + it: `npm run build:daemon` in `packages/env`, or `PI_ENV_DAEMON`), `mcp-instances.test.ts` (the supervisor with a + short backoff), `mcp-leases.test.ts`, `mcp-environments.test.ts`, `mcp-project.test.ts`, + `mcp-windows-command.test.ts`, `mcp-tools.test.ts`, `mcp-worker.test.ts` (fake provider: a project file, `tool_search`, + a call, a relocation). - **One-shot agents** (`plugins/agent`, port of Pi Mobile a9335ebea and 0e238b29f): code mode scripts' `tools.agent({ prompt, schema, label?, model?, thinkingLevel?, tools? })` (`callers: ["tools"]`, replay-safe). One conversation per call, owned by the tool task (abort cascades) and recorded in the task-scoped `p2.agent-run` doc, so a rerun diff --git a/apps/p2/package.json b/apps/p2/package.json index 5b5234a33..9640085d1 100644 --- a/apps/p2/package.json +++ b/apps/p2/package.json @@ -22,6 +22,8 @@ "@earendil-works/pi-ai": "1.5.0", "@earendil-works/pi-codemode": "1.5.0", "@earendil-works/pi-durable": "1.5.0", + "@earendil-works/pi-env": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", "@earendil-works/pi-schema": "1.5.0", "diff": "8.0.4", "esbuild": "0.28.2", diff --git a/apps/p2/plugins/envs/manager.ts b/apps/p2/plugins/envs/manager.ts index a77440e34..a08638870 100644 --- a/apps/p2/plugins/envs/manager.ts +++ b/apps/p2/plugins/envs/manager.ts @@ -6,12 +6,14 @@ import { promisify } from "node:util"; import type { FileInfo } from "@earendil-works/pi-durable/env"; import { acceptHostKey, + type Connection, connectSsh, forgetHostKey, HostKeyChangedError, HostKeyUnknownError, RemoteExecutionEnv, scanHostKey, + sshConnection, } from "@earendil-works/pi-env"; import { BACKGROUND_CONTEXT } from "@p2/app/v1"; import { remotePaths, value } from "@p2/app/v1/node"; @@ -62,6 +64,14 @@ export class EnvManager { return config; } + /** + * A pi-env connection to env `name` configured as `config`, over SSH with p2's key and known hosts, deploying the + * daemon pi-env ships. It starts at its first request; the caller closes it. + */ + connection(name: string, config: EnvConfig): Connection { + return sshConnection(sshOptions(this.directory, name, config)).connection; + } + async list(): Promise<EnvSummary[]> { const envs = await loadEnvs(this.directory); return Promise.all( diff --git a/apps/p2/plugins/mcp/config.ts b/apps/p2/plugins/mcp/config.ts new file mode 100644 index 000000000..4a94c4a2f --- /dev/null +++ b/apps/p2/plugins/mcp/config.ts @@ -0,0 +1,189 @@ +// The owner's MCP configuration, `<P2_DATA_DIR>/mcp.json` (a trust boundary): the shared `mcpServers` shape checked by +// pi-mcp, p2's own fields (`shared`, `runOn`) checked here, and the file followed while the server runs. + +import { join } from "node:path"; +import { NodeExecutionEnv } from "@earendil-works/pi-durable/env/node"; +import { + type McpHttpServerConfig, + type McpStdioServerConfig, + mcpNamespace, + validateMcpServerConfig, +} from "@earendil-works/pi-mcp"; +import { Observer, optionalText, stripBom, stripJsonComments } from "@p2/app/v1/node"; + +/** Where a stdio server runs: where the conversation works, on the p2 host, or on an environment of `envs.json`. */ +type RunOn = "location" | "host" | { readonly env: string }; + +/** One server of the owner's configuration, validated. */ +export type OwnerServer = + | { + readonly name: string; + readonly kind: "stdio"; + readonly config: McpStdioServerConfig; + /** `"enabled": false` keeps it off, unless a project's `mcp.json` turns it on where it applies. */ + readonly enabled: boolean; + /** One process for every session (`"shared": true`), else one per session and location. */ + readonly shared: boolean; + readonly runOn: RunOn; + } + | { + readonly name: string; + readonly kind: "http"; + readonly config: McpHttpServerConfig; + readonly enabled: boolean; + /** One connection for every session (the default), else one per session. */ + readonly shared: boolean; + }; + +/** The owner's servers and why entries were skipped. */ +export type OwnerConfig = { readonly servers: readonly OwnerServer[]; readonly errors: readonly string[] }; + +/** + * The file as last read: not yet, a valid configuration, or why it is not one (its readers keep the last valid servers, + * also from before this reader existed: a reload's new reader may first see a save caught half-way). + */ +type OwnerConfigRead = + | { readonly kind: "unread" } + | { readonly kind: "valid"; readonly config: OwnerConfig } + | { readonly kind: "invalid"; readonly failure: string }; + +/** + * The owner's configuration in `text` (`{ "mcpServers": { name: entry } }`, comments allowed): invalid entries are + * skipped and reported in `errors`; disabled ones are kept (a project may turn them on). Throws when the file itself is + * not such an object. + */ +export function parseOwnerConfig(text: string): OwnerConfig { + const servers: OwnerServer[] = []; + const errors: string[] = []; + const namespaces = new Map<string, string>(); + for (const [name, raw] of Object.entries(serverEntries(text, "mcp.json"))) { + const entry = ownerServer(name, raw); + if (entry.kind === "invalid") { + errors.push(entry.error); + continue; + } + // Tool names and code mode namespaces replace `-` by `_`: both spellings would collide. + const taken = namespaces.get(mcpNamespace(name)); + if (taken !== undefined) { + errors.push(`server "${name}": its tools would be named like those of "${taken}"`); + continue; + } + namespaces.set(mcpNamespace(name), name); + servers.push(entry.server); + } + return { servers, errors }; +} + +/** + * The `mcpServers` entries of `text`, a configuration file named `file` (comments allowed); throws when the file is not + * such an object. + */ +export function serverEntries(text: string, file: string): Record<string, unknown> { + let value: unknown; + try { + value = JSON.parse(stripJsonComments(stripBom(text))); + } catch { + throw new Error(`Invalid ${file} syntax`); + } + if (!isRecord(value)) throw new Error(`${file} must be an object`); + const entries = value.mcpServers ?? {}; + if (!isRecord(entries)) throw new Error(`${file}: "mcpServers" must be an object`); + return entries; +} + +/** One entry of the configuration, checked. */ +type Entry = + | { readonly kind: "server"; readonly server: OwnerServer } + | { readonly kind: "invalid"; readonly error: string }; + +/** One entry: the server, or why it is invalid. */ +function ownerServer(name: string, raw: unknown): Entry { + const invalid = (error: string): Entry => ({ kind: "invalid", error: `server "${name}": ${error}` }); + const result = validateMcpServerConfig(name, raw); + if (result.kind === "invalid") return { kind: "invalid", error: result.error }; + // The entry keeps fields pi-mcp does not know (p2's `shared` and `runOn` among them), unused from here on. + const { shared, runOn } = result.config; + const enabled = result.config.enabled !== false; + if (shared !== undefined && typeof shared !== "boolean") return invalid("shared must be a boolean"); + if (result.kind === "http") { + if (runOn !== undefined) return invalid("runOn is for stdio servers"); + return { kind: "server", server: { name, kind: "http", config: result.config, enabled, shared: shared ?? true } }; + } + const placement = runOnOf(runOn); + if (placement.kind === "invalid") + return invalid('runOn must be "location", "host" or { "env": "<name in envs.json>" }'); + if (shared === true && placement.kind === "set" && placement.runOn === "location") + return invalid('a shared server cannot run where each conversation works (use runOn "host" or an env)'); + const defaultRunOn: RunOn = shared ? "host" : "location"; + return { + kind: "server", + server: { + name, + kind: "stdio", + config: result.config, + enabled, + shared: shared ?? false, + runOn: placement.kind === "set" ? placement.runOn : defaultRunOn, + }, + }; +} + +/** `runOn` as configured: absent (the default applies), set, or invalid. */ +function runOnOf(value: unknown): { kind: "absent" } | { kind: "set"; runOn: RunOn } | { kind: "invalid" } { + if (value === undefined) return { kind: "absent" }; + if (value === "location" || value === "host") return { kind: "set", runOn: value }; + if (isRecord(value) && Object.keys(value).length === 1 && typeof value.env === "string" && value.env !== "") + return { kind: "set", runOn: { env: value.env } }; + return { kind: "invalid" }; +} + +/** Whether `value` is a JSON object. */ +export function isRecord(value: unknown): value is Record<string, unknown> { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * `value` with every `${NAME}` replaced by the p2 host's environment variable (the owner's configuration only); an + * unset variable throws, naming it. + */ +export function expandHostVariables(value: string): string { + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_match, name: string) => { + const variable = process.env[name]; + if (variable === undefined) throw new Error(`environment variable ${name} is not set`); + return variable; + }); +} + +/** `mcp.json`, watched (a missing file is no servers); one process reads it (the server's manager). */ +export class OwnerConfigFile { + /** Watches the file and reads it at each change; `start()` reads it the first time, `close()` stops. */ + readonly observer: Observer; + #read: OwnerConfigRead = { kind: "unread" }; + readonly #listeners = new Set<(read: OwnerConfigRead) => void>(); + + constructor(dataDir: string) { + const path = join(dataDir, "mcp.json"); + this.observer = new Observer( + [{ env: new NodeExecutionEnv({ cwd: dataDir }), targets: () => [{ path }] }], + async (retrying) => { + try { + this.#read = { kind: "valid", config: parseOwnerConfig((await optionalText(path)) ?? "{}") }; + } catch (error) { + // A save caught half-way is read once more before it counts. + if (!retrying) return true; + this.#read = { kind: "invalid", failure: error instanceof Error ? error.message : String(error) }; + } + for (const listener of this.#listeners) listener(this.#read); + return false; + }, + 2000, + ); + } + + /** `listener` gets the last read at once (once there is one), then every read's. */ + subscribe(listener: (read: OwnerConfigRead) => void): () => void { + this.#listeners.add(listener); + if (this.#read.kind !== "unread") listener(this.#read); + return () => this.#listeners.delete(listener); + } +} diff --git a/apps/p2/plugins/mcp/contract.ts b/apps/p2/plugins/mcp/contract.ts new file mode 100644 index 000000000..ab83e5f42 --- /dev/null +++ b/apps/p2/plugins/mcp/contract.ts @@ -0,0 +1,146 @@ +// MCP servers' contracts: the server's manager (host-internal: every MCP connection and process, by instance and lease) +// and its view per principal (`Mcp`: a session worker sets its locations and calls tools; the owner sees and restarts +// instances). + +import type { CallToolResult, ToolAnnotations } from "@earendil-works/pi-mcp"; +import { arr, int, nul, obj, oneOf, opt, type Static, str, union, unknown } from "@earendil-works/pi-schema"; +import { LocationSchema, literal, local, method, record, remote, serverHost, unsafe } from "@p2/app/v1"; + +/** An MCP tool as its server lists it (`tools/list`). */ +const ToolSchema = obj({ + name: str(), + title: opt(str()), + description: opt(str()), + inputSchema: unsafe<Record<string, unknown>>(unknown()), + outputSchema: opt(unsafe<Record<string, unknown>>(unknown())), + annotations: opt(unsafe<ToolAnnotations>(unknown())), +}); +/** An MCP tool as its server lists it: name, title, description, schemas and annotations. */ +export type McpTool = Static<typeof ToolSchema>; + +/** Where an instance runs: the p2 host, or an environment of `envs.json`. */ +const RunsOnSchema = union([literal("host"), obj({ env: str() })]); +/** Where an instance runs: `"host"` (the p2 host) or `{ env }` (an environment of `envs.json`). */ +export type RunsOn = Static<typeof RunsOnSchema>; + +/** One running (or failed) server process or connection. */ +export const InstanceSchema = obj({ + id: str(), + server: str(), + source: oneOf(["owner", "project"]), + /** The location it serves; null: shared by every session, or an HTTP server. */ + location: union([LocationSchema, nul()]), + runsOn: RunsOnSchema, + /** `sign-in`: the HTTP server asked for authorization. */ + status: oneOf(["starting", "connected", "sign-in", "failed"]), + error: opt(str()), + /** Kept while it reconnects. */ + tools: arr(ToolSchema), +}); +/** One instance as the manager lists it: its server, where it runs, its status, error and tools. */ +export type Instance = Static<typeof InstanceSchema>; + +/** The instances one session's lease at a location uses, and why entries of its project were refused. */ +const LeaseViewSchema = obj({ session: str(), location: LocationSchema, instances: arr(str()), errors: arr(str()) }); + +/** The largest project `mcp.json` a worker sends. */ +export const PROJECT_FILE_MAX_CHARS = 256 * 1024; + +/** + * A project's `<cwd>/.pi/mcp.json` at one of the locations, as the worker read it there: its text (checked by the + * manager: a trust boundary), or why it could not be read. + */ +const ProjectFileSchema = union([ + obj({ location: LocationSchema, text: str({ maxLength: PROJECT_FILE_MAX_CHARS }) }), + obj({ location: LocationSchema, error: str({ maxLength: 2000 }) }), +]); +/** A project's `mcp.json` at a location: its text, or why it could not be read. */ +export type ProjectFile = Static<typeof ProjectFileSchema>; + +/** Where a session's conversations work, and the project files found there (absent: none). */ +const Locations = { locations: arr(LocationSchema), projects: opt(arr(ProjectFileSchema)) }; + +/** `tools/call`'s result as the server sent it. */ +const CallResultSchema = unsafe<CallToolResult>({ type: "object" }); + +/** Tool arguments: any JSON object (a record of any values); the MCP server checks them. */ +const ArgsSchema = record(unknown()); + +/** What every worker request names: the session's worker it comes from (filled in by the view). */ +const Caller = { session: str(), generation: int(), incarnation: str() }; + +/** + * The MCP manager (server, host lifetime): the owner configuration, the instances and their leases. Every MCP process + * and connection is here; session workers reach it only through `Mcp`. + * + * Authority: a lease places servers only at a location on an environment the session's conversations work on as the + * server records them (the catalog's session location, the subagent store's own locations); the cwd there is the + * worker's to choose, as its agent can run commands anywhere on that machine. A project's servers are code the worker + * names (it reads the file), so this fences stale or confused workers to their machines, not a compromised worker + * process: it runs unsandboxed on the p2 host, its subagent store entries are its own reports, and its location tool + * moves the session to any configured environment. + */ +export const McpManager = local("p2.mcp.manager", { + version: 1, + params: {}, + members: { + instances: arr(InstanceSchema), + /** The leases of every session and location, with the instances each uses. */ + leases: arr(LeaseViewSchema), + /** Why entries of `mcp.json` were skipped (the leases' projects' errors are in `leases`). */ + errors: arr(str()), + /** + * Leases the configured servers (and the projects') at exactly `locations` for the caller's session (its leases + * elsewhere end); a newer generation's leases refuse an older generation. + */ + setLocations: method(obj({ ...Caller, ...Locations })), + /** Ends the leases of one worker generation (its view ended). */ + releaseLeases: method(obj({ session: str(), generation: int() })), + /** Calls `tool` of `server` at `location`, through the instance the lease there uses. */ + call: method( + obj({ ...Caller, location: LocationSchema, server: str(), tool: str(), args: ArgsSchema }), + CallResultSchema, + ), + restart: method(obj({ instance: str() })), + }, +}); + +/** + * MCP servers as a principal sees them: a session worker sets its locations and calls tools (its session, generation + * and incarnation from the link), the owner sees every instance with its errors and restarts them. Nobody else gets + * one. + */ +export const Mcp = remote("p2.mcp", { + version: 1, + params: {}, + hostAddress: serverHost, + variants: { + worker: { + state: { + /** The instances the session's leases use, shared ones included; `error` is the owner's only. */ + servers: arr(InstanceSchema), + /** Why the session's leases refused entries of its projects' `mcp.json` (or a location). */ + errors: arr(str()), + }, + methods: { + /** + * Where the session's conversations work now, with the project files there: its servers are leased there, + * and only there. + */ + setLocations: method(obj(Locations)), + call: method( + obj({ location: LocationSchema, server: str(), tool: str(), args: ArgsSchema }), + CallResultSchema, + ), + }, + }, + owner: { + /** `errors`: the owner's `mcp.json`'s, then every lease's (by session). */ + state: { instances: arr(InstanceSchema), errors: arr(str()) }, + methods: { restart: method(obj({ instance: str() })) }, + }, + }, +}); + +/** The worker's Durable extension of MCP tools. */ +export const MCP_EXTENSION = "mcp"; diff --git a/apps/p2/plugins/mcp/environments.ts b/apps/p2/plugins/mcp/environments.ts new file mode 100644 index 000000000..ac882efeb --- /dev/null +++ b/apps/p2/plugins/mcp/environments.ts @@ -0,0 +1,44 @@ +// The supervisor's pi-env connections to environments: one per environment and configuration revision, shared by every +// instance running there and closed with the last one. A connection lost meanwhile starts again at its next request +// (pi-env); the daemon killed the programs it ran, and their instances reconnect. + +import type { Connection } from "@earendil-works/pi-env"; +import { envRevision } from "@p2/app/v1/node"; +import type { EnvConfig } from "../envs/config.ts"; + +/** A connection to environment `name` configured as `config`, which starts at its first request. */ +export type ConnectEnv = (name: string, config: EnvConfig) => Connection; + +/** One instance's use of an environment's connection; released once, when the instance closes. */ +export type EnvLease = { readonly connection: Connection; release(): void }; + +/** The open connections by environment and revision, with how many instances use each. */ +export class EnvConnections { + readonly #connect: ConnectEnv; + readonly #open = new Map<string, { readonly connection: Connection; users: number }>(); + + constructor(connect: ConnectEnv) { + this.#connect = connect; + } + + /** The connection to `name` as configured now (opened for its first user); release it once done. */ + acquire(name: string, config: EnvConfig): EnvLease { + const key = JSON.stringify([name, envRevision(config)]); + let entry = this.#open.get(key); + if (entry === undefined) { + entry = { connection: this.#connect(name, config), users: 0 }; + this.#open.set(key, entry); + } + const used = entry; + used.users++; + return { + connection: used.connection, + release: () => { + used.users--; + if (used.users > 0) return; + this.#open.delete(key); + used.connection.close(); + }, + }; + } +} diff --git a/apps/p2/plugins/mcp/instances.ts b/apps/p2/plugins/mcp/instances.ts new file mode 100644 index 000000000..7cd9dc316 --- /dev/null +++ b/apps/p2/plugins/mcp/instances.ts @@ -0,0 +1,346 @@ +// The MCP supervisor, a native of the manager: one client per instance the placement needs, started when a lease first +// needs it and closed when nothing needs it any more (a shared instance stays while its server is configured). A lost +// connection (a crashed server, or an environment's connection whose daemon killed it) reconnects with backoff, +// keeping its tools listed; a server's tool-list change refreshes its tools. + +import { + type CallToolResult, + McpAuthRequiredError, + McpClient, + type McpTransport, + PipeTransport, + StdioTransport, + type Tool, +} from "@earendil-works/pi-mcp"; +import { McpOAuthAuthorizationRequiredError } from "@earendil-works/pi-mcp/oauth"; +import { PluginError } from "@p2/app/v1"; +import type { Instance, McpTool } from "./contract.ts"; +import { type ConnectEnv, EnvConnections } from "./environments.ts"; +import type { InstanceSpec, Placement } from "./placement.ts"; +import { DaemonTooOldError, type Opened, type Transports, transportsFor } from "./transports.ts"; + +/** How the p2 server names itself to MCP servers (`initialize`'s `clientInfo`). */ +export type ClientInfo = { readonly name: string; readonly version: string }; + +/** Reconnect waits: the first, the cap of their doubling, and how long a connection lasts to start over at the first. */ +export type Backoff = { readonly firstMs: number; readonly maxMs: number; readonly stableMs: number }; + +/** The manager's backoff: 1 s doubling to 60 s, from 1 s again after a connection that was up for a minute. */ +export const RECONNECT: Backoff = { firstMs: 1000, maxMs: 60_000, stableMs: 60_000 }; + +/** The running instances; subscribers hear of every change of what `list` returns. */ +export class Instances { + readonly #client: ClientInfo; + readonly #backoff: Backoff; + /** Tools last listed per instance ID (a reload's previous supervisor): listed until the server answers again. */ + readonly #seed: ReadonlyMap<string, readonly McpTool[]>; + readonly #envs: EnvConnections; + readonly #running = new Map<string, Connection>(); + readonly #listeners = new Set<() => void>(); + #closed = false; + + /** `connect` opens the connection to an environment its instances share. */ + constructor(client: ClientInfo, backoff: Backoff, seed: readonly Instance[], connect: ConnectEnv) { + this.#client = client; + this.#backoff = backoff; + this.#seed = new Map(seed.map((instance) => [instance.id, instance.tools])); + this.#envs = new EnvConnections(connect); + } + + /** `listener` is called after every change of `list`; the result stops it. */ + subscribe(listener: () => void): () => void { + this.#listeners.add(listener); + return () => this.#listeners.delete(listener); + } + + /** Starts the instances `placement`'s leases use and closes the others, except shared ones still configured. */ + reconcile(placement: Placement): void { + if (this.#closed) return; + const needed = new Set(placement.leases.flatMap((lease) => [...lease.servers.values()])); + for (const id of placement.kept) if (this.#running.has(id)) needed.add(id); + let changed = false; + for (const [id, connection] of this.#running) + if (!needed.has(id)) { + this.#running.delete(id); + void connection.close(); + changed = true; + } + for (const id of needed) + if (!this.#running.has(id)) { + // Every leased ID comes from the same placement's instances. + const spec = placement.instances.get(id)!; + const transports = transportsFor(spec.connect, this.#envs); + const connection = new Connection( + spec, + transports, + this.#client, + this.#backoff, + this.#seed.get(id) ?? [], + () => this.#changed(), + ); + this.#running.set(id, connection); + connection.start(); + changed = true; + } + if (changed) this.#changed(); + } + + /** The instances as the manager shows them, by server name, then ID. */ + list(): Instance[] { + return [...this.#running.values()] + .map((connection) => connection.view()) + .sort((a, b) => (a.server === b.server ? (a.id < b.id ? -1 : 1) : a.server < b.server ? -1 : 1)); + } + + /** Calls `tool` on instance `id`; throws while it is not connected. */ + async call(id: string, tool: string, args: Record<string, unknown>, signal: AbortSignal): Promise<CallToolResult> { + const connection = this.#running.get(id); + if (connection === undefined) throw new Error("This MCP server is not running"); + return await connection.call(tool, args, signal); + } + + /** Connects instance `id` again now, its backoff reset (a connect under way is closed first). */ + restart(id: string): void { + const connection = this.#running.get(id); + if (connection === undefined) throw new PluginError("not_found", `No MCP server instance ${id}`); + connection.restart(); + } + + /** Closes every instance (stdio servers get MCP's shutdown: stdin closed, then killed) and so every connection. */ + async close(): Promise<void> { + this.#closed = true; + const connections = [...this.#running.values()]; + this.#running.clear(); + await Promise.allSettled(connections.map((connection) => connection.close())); + } + + #changed(): void { + for (const listener of [...this.#listeners]) listener(); + } +} + +/** How much of a stdio server's stderr the owner's error shows. */ +const STDERR_TAIL_CHARS = 300; + +/** + * Where one connection is: not trying (before its start, unplaced, failed for good, waiting for a sign-in), opening its + * transport (an environment's connection may start first), connecting, connected, waiting to reconnect, or closed. + */ +type State = + | { readonly kind: "idle" } + | { readonly kind: "opening" } + | { readonly kind: "connecting"; readonly client: McpClient } + | { readonly kind: "connected"; readonly client: McpClient; readonly since: number } + | { readonly kind: "waiting"; readonly timer: ReturnType<typeof setTimeout> } + | { readonly kind: "closed" }; + +/** Why the last attempt failed: the message for everybody, the stderr tail for the owner only. */ +type Failure = { readonly message: string; readonly stderr?: string }; + +/** One instance: its connection state, and its status, failure and tools as listed. */ +class Connection { + readonly #spec: InstanceSpec; + readonly #transports: Transports; + readonly #info: ClientInfo; + readonly #backoff: Backoff; + readonly #changed: () => void; + #state: State = { kind: "idle" }; + #status: Instance["status"] = "starting"; + #failure: Failure | undefined; + #tools: readonly McpTool[]; + #delayMs: number; + + constructor( + spec: InstanceSpec, + transports: Transports, + info: ClientInfo, + backoff: Backoff, + tools: readonly McpTool[], + changed: () => void, + ) { + this.#spec = spec; + this.#transports = transports; + this.#info = info; + this.#backoff = backoff; + this.#tools = tools; + this.#changed = changed; + this.#delayMs = backoff.firstMs; + } + + start(): void { + if (this.#spec.connect.kind === "unplaced") this.#show("failed", { message: this.#spec.connect.reason }); + else void this.#connect(); + } + + view(): Instance { + const { id, server, source, location, runsOn } = this.#spec; + const failure = this.#failure; + return { + id, + server, + source, + location, + runsOn, + status: this.#status, + ...(failure === undefined + ? {} + : { error: failure.stderr ? `${failure.message} (stderr: ${failure.stderr})` : failure.message }), + tools: [...this.#tools], + }; + } + + /** The error says why it is not connected, never the server's stderr (the model reads it). */ + async call(tool: string, args: Record<string, unknown>, signal: AbortSignal): Promise<CallToolResult> { + if (this.#state.kind !== "connected") { + const why = this.#failure === undefined ? "" : `: ${this.#failure.message}`; + throw new Error(`MCP server "${this.#spec.server}" is not connected${why}`); + } + return await this.#state.client.callTool(tool, args, { signal, timeoutMs: this.#spec.timeoutMs }); + } + + restart(): void { + if (this.#spec.connect.kind === "unplaced" || this.#state.kind === "closed") return; + this.#stop(); + this.#delayMs = this.#backoff.firstMs; + void this.#connect(); + } + + /** Closes the client, then lets go of what its transports held (an environment's connection, maybe its last user). */ + async close(): Promise<void> { + const closing = this.#stop(); + this.#state = { kind: "closed" }; + await closing; + this.#transports.release(); + } + + /** Ends what the state holds (a connect under way, a client, a timer) and goes idle. */ + #stop(): Promise<void> { + const state = this.#state; + this.#state = { kind: "idle" }; + switch (state.kind) { + case "connecting": + case "connected": + return state.client.close().catch(() => {}); + case "waiting": + clearTimeout(state.timer); + return Promise.resolve(); + default: + return Promise.resolve(); + } + } + + async #connect(): Promise<void> { + const opening: State = { kind: "opening" }; + this.#state = opening; + this.#show("starting", this.#failure); + let opened: Opened; + try { + opened = await this.#transports.open(); + } catch (error) { + if (this.#state === opening) this.#failed(error, undefined); + return; + } + // Closed or restarted meanwhile: the transport never started, so there is nothing to close. + if (this.#state === opening) await this.#initialize(opened); + } + + async #initialize({ transport, roots }: Opened): Promise<void> { + const client = new McpClient({ + ...this.#info, + ...(this.#spec.timeoutMs === undefined ? {} : { requestTimeoutMs: this.#spec.timeoutMs }), + ...(roots.length ? { roots: roots.map((uri) => ({ uri })) } : {}), + }); + const state: State = { kind: "connecting", client }; + this.#state = state; + try { + // Only a connection that was up is lost; a failed connect closes its client too. + client.onClose(() => { + if (this.#state.kind === "connected" && this.#state.client === client) this.#lost(transport); + }); + client.onNotification("notifications/tools/list_changed", () => void this.#refresh(client)); + await client.connect(transport); + const tools = await client.listTools(); + // Closed or restarted meanwhile: that already closed this client. + if (this.#state !== state) return; + this.#state = { kind: "connected", client, since: Date.now() }; + this.#tools = tools.map(toolOf); + this.#show("connected", undefined); + } catch (error) { + await client.close().catch(() => {}); + if (this.#state === state) this.#failed(error, transport); + } + } + + /** A connect failed: wait for a sign-in, stay failed (a daemon too old), or try again after the backoff. */ + #failed(error: unknown, transport: McpTransport | undefined): void { + if (error instanceof McpAuthRequiredError || error instanceof McpOAuthAuthorizationRequiredError) { + this.#state = { kind: "idle" }; + this.#show("sign-in", { message: "This server needs a sign-in" }); + return; + } + if (error instanceof DaemonTooOldError) { + this.#state = { kind: "idle" }; + this.#show("failed", { message: error.message }); + return; + } + this.#show("failed", failureOf(error, transport)); + this.#wait(); + } + + /** The connection dropped (a stdio server exited, say): reconnect after the backoff, tools still listed. */ + #lost(transport: McpTransport): void { + if (this.#state.kind === "connected" && Date.now() - this.#state.since >= this.#backoff.stableMs) + this.#delayMs = this.#backoff.firstMs; + this.#show("starting", failureOf(new Error("the connection was lost"), transport)); + this.#wait(); + } + + #wait(): void { + const timer = setTimeout(() => { + if (this.#state === state) void this.#connect(); + }, this.#delayMs); + timer.unref(); + const state: State = { kind: "waiting", timer }; + this.#state = state; + this.#delayMs = Math.min(this.#delayMs * 2, this.#backoff.maxMs); + } + + async #refresh(client: McpClient): Promise<void> { + try { + const tools = await client.listTools(); + if (this.#state.kind !== "connected" || this.#state.client !== client) return; + this.#tools = tools.map(toolOf); + this.#changed(); + } catch (error) { + console.error(`p2 mcp: refreshing the tools of "${this.#spec.server}" failed:`, error); + } + } + + #show(status: Instance["status"], failure: Failure | undefined): void { + this.#status = status; + this.#failure = failure; + this.#changed(); + } +} + +/** A tool as instances list it: plain JSON, without the fields p2 does not use. */ +function toolOf(tool: Tool): McpTool { + return { + name: tool.name, + ...(tool.title === undefined ? {} : { title: tool.title }), + ...(tool.description === undefined ? {} : { description: tool.description }), + inputSchema: tool.inputSchema, + ...(tool.outputSchema === undefined ? {} : { outputSchema: tool.outputSchema }), + ...(tool.annotations === undefined ? {} : { annotations: tool.annotations }), + }; +} + +/** `error`'s message, and the end of a stdio server's stderr (on the p2 host or an environment). */ +function failureOf(error: unknown, transport: McpTransport | undefined): Failure { + const message = error instanceof Error ? error.message : String(error); + const stderr = + transport instanceof StdioTransport || transport instanceof PipeTransport + ? transport.stderr.trim().slice(-STDERR_TAIL_CHARS) + : ""; + return stderr ? { message, stderr } : { message }; +} diff --git a/apps/p2/plugins/mcp/leases.ts b/apps/p2/plugins/mcp/leases.ts new file mode 100644 index 000000000..fe9221f01 --- /dev/null +++ b/apps/p2/plugins/mcp/leases.ts @@ -0,0 +1,78 @@ +// The rules of MCP leases, as pure functions over the lease list: a session worker sets the locations it works at for +// its generation (a newer generation takes the session over, an older one is refused), its calls go through the lease +// at a location only from the incarnation that set it, and its view's end releases its generation's leases. + +import { type Location, sameLocation } from "@p2/app/v1"; + +/** The project configuration at a location as the worker read it: none, the file's text, or why it could not. */ +export type ProjectSource = + | { readonly kind: "none" } + | { readonly kind: "file"; readonly text: string } + | { readonly kind: "unreadable"; readonly error: string }; + +/** + * A session worker's lease on the servers at one location, from the generation and incarnation that set it, with the + * project configuration found there. + */ +export type Lease = { + readonly session: string; + readonly location: Location; + readonly project: ProjectSource; + readonly generation: number; + readonly incarnation: string; +}; + +/** A location to lease, with its project configuration. */ +export type LeasePlace = { readonly location: Location; readonly project: ProjectSource }; + +/** Who asks: one incarnation of one generation of a session's worker. */ +export type LeaseCaller = { readonly session: string; readonly generation: number; readonly incarnation: string }; + +/** `setLocations`'s outcome: the new lease list, or why the caller may not set it. */ +export type LeasesSet = + | { readonly kind: "set"; readonly leases: Lease[] } + | { readonly kind: "refused"; readonly reason: string }; + +/** The lease `call` goes through: held by the caller, none at that location, or another incarnation's. */ +export type LeaseHeld = + | { readonly kind: "held"; readonly lease: Lease } + | { readonly kind: "none" } + | { readonly kind: "stale" }; + +/** + * `caller`'s session leases exactly `places` from now on, for `caller`'s generation and incarnation (of a location + * given twice, the first counts): its leases elsewhere end, those of older generations (a worker that is gone) are + * taken over. A newer generation's lease refuses. + */ +export function setLocations(leases: readonly Lease[], caller: LeaseCaller, places: readonly LeasePlace[]): LeasesSet { + const own = leases.filter((lease) => lease.session === caller.session); + if (own.some((lease) => lease.generation > caller.generation)) + return { kind: "refused", reason: "A newer worker of this session holds its MCP servers" }; + const { session, generation, incarnation } = caller; + const wanted = places.filter( + (place, index) => places.findIndex((other) => sameLocation(other.location, place.location)) === index, + ); + return { + kind: "set", + leases: [ + ...leases.filter((lease) => lease.session !== session), + ...wanted.map(({ location, project }) => ({ session, location, project, generation, incarnation })), + ], + }; +} + +/** The lease `caller` calls through at `location`. */ +export function heldLease(leases: readonly Lease[], caller: LeaseCaller, location: Location): LeaseHeld { + const lease = leases.find( + (candidate) => candidate.session === caller.session && sameLocation(candidate.location, location), + ); + if (lease === undefined) return { kind: "none" }; + return lease.incarnation === caller.incarnation && lease.generation === caller.generation + ? { kind: "held", lease } + : { kind: "stale" }; +} + +/** `leases` without those of `session`'s `generation` (its view ended). */ +export function releaseGeneration(leases: readonly Lease[], session: string, generation: number): Lease[] { + return leases.filter((lease) => lease.session !== session || lease.generation !== generation); +} diff --git a/apps/p2/plugins/mcp/package.json b/apps/p2/plugins/mcp/package.json new file mode 100644 index 000000000..08d43cd20 --- /dev/null +++ b/apps/p2/plugins/mcp/package.json @@ -0,0 +1,5 @@ +{ + "name": "mcp", + "private": true, + "type": "module" +} diff --git a/apps/p2/plugins/mcp/placement.ts b/apps/p2/plugins/mcp/placement.ts new file mode 100644 index 000000000..73bfafb0f --- /dev/null +++ b/apps/p2/plugins/mcp/placement.ts @@ -0,0 +1,249 @@ +// Where each MCP instance runs and which leases share it: for every session's lease at a location, the owner's servers +// it enables (one process per session and location, or one for everybody) and its project's own servers (always at +// that location), each with how it connects: a program on the p2 host or on an environment, or an HTTP server. + +import { createHash } from "node:crypto"; +import { homedir } from "node:os"; +import { join, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import type { McpStdioServerConfig } from "@earendil-works/pi-mcp"; +import { type Location, locationText } from "@p2/app/v1"; +import { envRevision } from "@p2/app/v1/node"; +import type { EnvConfig } from "../envs/config.ts"; +import type { OwnerConfig, OwnerServer } from "./config.ts"; +import type { RunsOn } from "./contract.ts"; +import type { Lease } from "./leases.ts"; +import { type ProjectConfig, type ProjectServer, parseProjectConfig } from "./project.ts"; + +/** A program on an environment, started through its pi-env daemon; paths resolve once the daemon tells its system. */ +export type RemoteConnect = { + readonly kind: "remote"; + /** The environment's name in envs.json, and its configuration there. */ + readonly environment: string; + readonly config: EnvConfig; + readonly command: string; + readonly args: readonly string[]; + readonly env: Readonly<Record<string, string>>; + /** As configured (`.` when not): relative to `base`, `~` the remote home. */ + readonly cwd: string; + /** The location's cwd when the location is on this environment (also the client's root), else null: the home. */ + readonly base: string | null; + /** `${NAME}` in `env` values comes from the p2 host's environment (the owner's servers only). */ + readonly hostVariables: boolean; +}; + +/** How an instance connects: a program on the p2 host or on an environment, an HTTP server, or why it cannot run. */ +export type Connect = + | { + readonly kind: "host"; + readonly command: string; + readonly args: readonly string[]; + readonly env: Readonly<Record<string, string>>; + readonly cwd: string; + /** `file://` URLs offered as the client's roots (an instance on its location's machine only). */ + readonly roots: readonly string[]; + /** `${NAME}` in `env` values comes from the p2 host's environment (the owner's servers only). */ + readonly hostVariables: boolean; + } + | RemoteConnect + | { readonly kind: "http"; readonly url: string; readonly headers: Readonly<Record<string, string>> } + | { readonly kind: "unplaced"; readonly reason: string }; + +/** One instance to run: its identity and where and how it runs. Equal configuration and key give an equal ID. */ +export type InstanceSpec = { + readonly id: string; + readonly server: string; + readonly source: "owner" | "project"; + readonly location: Location | null; + readonly runsOn: RunsOn; + readonly connect: Connect; + /** Per-request timeout from the configuration, in milliseconds. */ + readonly timeoutMs: number | undefined; +}; + +/** One lease placed: the instance of each server it uses, by server name, and why entries of its project were refused. */ +export type PlacedLease = { + readonly lease: Lease; + readonly servers: ReadonlyMap<string, string>; + readonly errors: readonly string[]; +}; + +/** What the leases need: the instances each lease uses, and the shared instances that may stay without one. */ +export type Placement = { + readonly instances: ReadonlyMap<string, InstanceSpec>; + readonly leases: readonly PlacedLease[]; + /** Shared instances of enabled owner servers: they stay running while configured, leased or not. */ + readonly kept: ReadonlySet<string>; +}; + +/** + * The environments each session's conversations work on, by session, as the server records them (the session's own, + * its subagents' own locations'); null is the p2 host. + */ +export type WorkEnvironments = Readonly<Record<string, readonly (string | null)[]>>; + +/** + * The instances `config`'s servers and the leases' projects need, on the environments of `envs`. A lease at a location + * on an environment its session does not work on (`work`) gets no servers: a worker can only lease where its + * conversations can already run commands. + */ +export function place( + config: OwnerConfig, + leases: readonly Lease[], + envs: Readonly<Record<string, EnvConfig>>, + work: WorkEnvironments, +): Placement { + const instances = new Map<string, InstanceSpec>(); + const kept = new Set<string>(); + for (const server of config.servers) + if (server.enabled && server.shared) { + const spec = ownerSpec(server, undefined, envs); + kept.add(spec.id); + instances.set(spec.id, spec); + } + const placed = leases.map((lease): PlacedLease => { + const where = locationText(lease.location); + if (!(work[lease.session] ?? []).includes(lease.location.env)) + return { lease, servers: new Map(), errors: [`${where}: not where this session works`] }; + const project = projectOf(lease, config); + const servers = new Map<string, string>(); + const use = (spec: InstanceSpec): void => { + instances.set(spec.id, spec); + servers.set(spec.server, spec.id); + }; + for (const server of config.servers) + if (project.enabled.get(server.name) ?? server.enabled) use(ownerSpec(server, lease, envs)); + for (const server of project.servers) use(projectSpec(server, lease, envs)); + return { lease, servers, errors: project.errors.map((error) => `${where}: ${error}`) }; + }); + return { instances, leases: placed, kept }; +} + +/** The project configuration the lease's worker sent, checked. */ +function projectOf(lease: Lease, config: OwnerConfig): ProjectConfig { + const { project } = lease; + if (project.kind === "file") return parseProjectConfig(project.text, config); + return { + servers: [], + enabled: new Map(), + errors: project.kind === "unreadable" ? [`.pi/mcp.json: ${project.error}`] : [], + }; +} + +/** The instance of owner server `server` that `lease` uses (shared: everybody's, `lease` unused). */ +function ownerSpec( + server: OwnerServer, + lease: Lease | undefined, + envs: Readonly<Record<string, EnvConfig>>, +): InstanceSpec { + const timeoutMs = server.config.timeout === undefined ? undefined : server.config.timeout * 1000; + const owned = server.shared ? undefined : lease; + if (server.kind === "http") { + const { url, headers = {} } = server.config; + // Unshared HTTP servers are one per session, wherever its conversations work. + const key = ["http", server.name, owned?.session ?? null, hashOf({ url, headers, timeoutMs: timeoutMs ?? null })]; + return { + id: idOf(key), + server: server.name, + source: "owner", + location: null, + runsOn: "host", + connect: { kind: "http", url, headers }, + timeoutMs, + }; + } + const at = owned?.location ?? null; + const runsOn: RunsOn = server.runOn !== "location" ? server.runOn : at?.env ? { env: at.env } : "host"; + const placed = { server: server.name, source: "owner" as const, session: owned?.session ?? null, location: at }; + return stdioSpec(placed, runsOn, server.config, timeoutMs, envs); +} + +/** The instance of project server `server` at `lease`'s location: always there, for that session alone. */ +function projectSpec(server: ProjectServer, lease: Lease, envs: Readonly<Record<string, EnvConfig>>): InstanceSpec { + const timeoutMs = server.config.timeout === undefined ? undefined : server.config.timeout * 1000; + const { session, location } = lease; + const runsOn: RunsOn = location.env === null ? "host" : { env: location.env }; + return stdioSpec( + { server: server.name, source: "project", session, location }, + runsOn, + server.config, + timeoutMs, + envs, + ); +} + +/** Who a stdio instance is for: its server, the configuration it comes from, and its session and location if unshared. */ +type StdioPlaced = { + readonly server: string; + readonly source: "owner" | "project"; + readonly session: string | null; + readonly location: Location | null; +}; + +/** A stdio instance on `runsOn`; on its location's machine its cwd is the base of relative paths and the root. */ +function stdioSpec( + { server, source, session, location }: StdioPlaced, + runsOn: RunsOn, + config: McpStdioServerConfig, + timeoutMs: number | undefined, + envs: Readonly<Record<string, EnvConfig>>, +): InstanceSpec { + const { command, args = [], env = {}, cwd } = config; + const hash = hashOf({ command, args, env, cwd: cwd ?? null, timeoutMs: timeoutMs ?? null }); + const owner = session === null ? null : [session, location]; + const machine = runsOn === "host" ? null : runsOn.env; + const base = location !== null && location.env === machine ? location.cwd : null; + const hostVariables = source === "owner"; + if (runsOn === "host") { + const key = ["stdio", source, server, owner, runsOn, hash]; + const spec = { id: idOf(key), server, source, location, runsOn, timeoutMs }; + return { + ...spec, + connect: { + kind: "host", + command: expandHome(command), + args: args.map(expandHome), + env, + cwd: resolve(base ?? homedir(), expandHome(cwd ?? ".")), + roots: base === null ? [] : [pathToFileURL(base).href], + hostVariables, + }, + }; + } + const envConfig = Object.hasOwn(envs, runsOn.env) ? envs[runsOn.env] : undefined; + // An edited environment is another connection: its instances start again on it. + const key = ["stdio", source, server, owner, runsOn, envConfig === undefined ? null : envRevision(envConfig), hash]; + const spec = { id: idOf(key), server, source, location, runsOn, timeoutMs }; + if (envConfig === undefined) + return { ...spec, connect: { kind: "unplaced", reason: `environment "${runsOn.env}" is not in envs.json` } }; + return { + ...spec, + connect: { + kind: "remote", + environment: runsOn.env, + config: envConfig, + command, + args, + env, + cwd: cwd ?? ".", + base, + hostVariables, + }, + }; +} + +/** `~` and `~/…` name the home directory, as in a shell (and in pi). */ +function expandHome(value: string): string { + if (value === "~") return homedir(); + return value.startsWith("~/") ? join(homedir(), value.slice(2)) : value; +} + +/** The configuration hash: what makes another process or connection (command, args, env, cwd; URL and headers; the timeout). */ +function hashOf(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16); +} + +/** An instance's ID from its key. */ +function idOf(key: unknown): string { + return createHash("sha256").update(JSON.stringify(key)).digest("hex").slice(0, 16); +} diff --git a/apps/p2/plugins/mcp/project.ts b/apps/p2/plugins/mcp/project.ts new file mode 100644 index 000000000..198382aa0 --- /dev/null +++ b/apps/p2/plugins/mcp/project.ts @@ -0,0 +1,94 @@ +// A project's MCP configuration, `<cwd>/.pi/mcp.json` as a session's worker read it at one location (a trust boundary: +// a repository wrote it, and the agent can edit it). It may add stdio servers, which run at that location only, and +// turn the owner's servers on or off there; everything else is refused and reported. + +import { + isMcpServerOverride, + type McpStdioServerConfig, + mcpNamespace, + validateMcpServerConfig, +} from "@earendil-works/pi-mcp"; +import { isRecord, type OwnerConfig, serverEntries } from "./config.ts"; + +/** One stdio server a project adds, validated; it runs where the project is. */ +export type ProjectServer = { readonly name: string; readonly config: McpStdioServerConfig }; + +/** A project's own servers, its choice of `enabled` per owner server, and why entries were refused. */ +export type ProjectConfig = { + readonly servers: readonly ProjectServer[]; + readonly enabled: ReadonlyMap<string, boolean>; + readonly errors: readonly string[]; +}; + +/** + * The project configuration in `text`, checked against `owner`'s servers: + * + * - an entry without `command`, `url` or `type` overrides only `enabled` of the owner's server of that name (as in pi); + * - an entry defining a server the owner defines is refused (the owner's entry wins); + * - stdio servers only, without `runOn` and `shared`: a project's server runs at its location, where the agent already + * runs commands, never on the p2 host or another machine for a repository's sake; + * - no `auth` or `oauth`: a repository does not choose where a credential goes; + * - `${NAME}` is never expanded: `env` values reach the program as written, in the location's own environment. + */ +export function parseProjectConfig(text: string, owner: OwnerConfig): ProjectConfig { + let entries: Record<string, unknown>; + try { + entries = serverEntries(text, ".pi/mcp.json"); + } catch (error) { + return { servers: [], enabled: new Map(), errors: [error instanceof Error ? error.message : String(error)] }; + } + const servers: ProjectServer[] = []; + const enabled = new Map<string, boolean>(); + const errors: string[] = []; + const namespaces = new Map(owner.servers.map((server) => [mcpNamespace(server.name), server.name])); + for (const [name, raw] of Object.entries(entries)) { + const entry = projectEntry(name, raw, owner); + if (entry.kind === "invalid") errors.push(`.pi/mcp.json: ${entry.error}`); + else if (entry.kind === "override" && entry.enabled !== undefined) enabled.set(name, entry.enabled); + if (entry.kind !== "server") continue; + const taken = namespaces.get(mcpNamespace(name)); + if (taken !== undefined) { + errors.push(`.pi/mcp.json: server "${name}": its tools would be named like those of "${taken}"`); + continue; + } + namespaces.set(mcpNamespace(name), name); + servers.push(entry.server); + } + return { servers, enabled, errors }; +} + +/** One entry of a project's configuration: a server, an override, disabled, or why it is refused. */ +type Entry = + | { readonly kind: "server"; readonly server: ProjectServer } + | { readonly kind: "override"; readonly enabled: boolean | undefined } + | { readonly kind: "disabled" } + | { readonly kind: "invalid"; readonly error: string }; + +/** The owner's fields a project may not set, and why. */ +const OWNER_ONLY: readonly (readonly [string, string])[] = [ + ["runOn", "a project's server runs where the project is"], + ["shared", "a project's server runs for its session alone"], + ["auth", "credentials are for the owner's mcp.json"], + ["oauth", "credentials are for the owner's mcp.json"], +]; + +/** One entry checked against the rules of `parseProjectConfig`. */ +function projectEntry(name: string, raw: unknown, owner: OwnerConfig): Entry { + const refused = (why: string): Entry => ({ kind: "invalid", error: `server "${name}": ${why}` }); + const defined = owner.servers.some((server) => server.name === name); + if (isRecord(raw) && isMcpServerOverride(raw)) { + if (!defined) return refused('needs "command", or a server of the owner\'s mcp.json to override'); + if (Object.keys(raw).some((key) => key !== "enabled")) return refused('an override can only set "enabled"'); + if (raw.enabled !== undefined && typeof raw.enabled !== "boolean") return refused("enabled must be a boolean"); + return { kind: "override", enabled: raw.enabled }; + } + if (defined) return refused('the owner\'s mcp.json defines it; a project can only set its "enabled"'); + const result = validateMcpServerConfig(name, raw); + if (result.kind === "invalid") return { kind: "invalid", error: result.error }; + if (result.kind === "http") + return refused("HTTP servers are for the owner's mcp.json (the p2 host would make the requests)"); + for (const [field, why] of OWNER_ONLY) + if (result.config[field] !== undefined) return refused(`"${field}" is for the owner's mcp.json: ${why}`); + if (result.config.enabled === false) return { kind: "disabled" }; + return { kind: "server", server: { name, config: result.config } }; +} diff --git a/apps/p2/plugins/mcp/server.ts b/apps/p2/plugins/mcp/server.ts new file mode 100644 index 000000000..1b74d23d6 --- /dev/null +++ b/apps/p2/plugins/mcp/server.ts @@ -0,0 +1,388 @@ +// The MCP plugin's server facet: the manager (the owner's configuration, the leases of session workers with their +// projects' configurations, the environments and where sessions work, the supervisor of instances) and its view per +// principal: a session worker's (its locations and calls, fenced to the session's current worker; its leases released +// when the view ends) and the owner's (every instance with its errors, restart). + +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { arr, int, nul, obj, str, union, unknown } from "@earendil-works/pi-schema"; +import { + type Ctx, + defineFacet, + defineTypedState, + deriveTyped, + type Facet, + type Factory, + forbidden, + type HandleOf, + isOwner, + isWorker, + type Location, + LocationSchema, + literal, + locationText, + type Native, + nativeOf, + type Ref, + type Self, + sameLocation, + signalOf, + unsafe, + until, + untilAborted, + type Watchable, + type WorkerPrincipal, +} from "@p2/app/v1"; +import { Hosts, Server, type WorkerState } from "@p2/app/v1/server"; +import type { EnvConfig } from "../envs/config.ts"; +import { EnvManagers, EnvService } from "../envs/contract.ts"; +import { Catalog, type Session } from "../sessions/contract.ts"; +import { type Child, ChildStore } from "../subagents/contract.ts"; +import { type OwnerConfig, OwnerConfigFile, type OwnerServer } from "./config.ts"; +import { InstanceSchema, Mcp, McpManager, type ProjectFile } from "./contract.ts"; +import type { ConnectEnv } from "./environments.ts"; +import { type ClientInfo, Instances, RECONNECT } from "./instances.ts"; +import { heldLease, type Lease, type ProjectSource, releaseGeneration, setLocations } from "./leases.ts"; +import { type Placement, place, type WorkEnvironments } from "./placement.ts"; + +/** A tool call as the worker's view takes it; the manager's adds the caller. */ +type CallArgs = { location: Location; server: string; tool: string; args: Record<string, unknown> }; +/** Who calls the manager: a session's worker generation and incarnation. */ +type Caller = { session: string; generation: number; incarnation: string }; +/** Where a session works, and its project files there. */ +type LocationsArgs = { locations: Location[]; projects?: ProjectFile[] }; + +/** How the manager reaches environments, given its object (whose `EnvService` it declared). */ +export type EnvConnector = (self: Self) => ConnectEnv; + +/** How long a worker's view outlives its last reference: a worker object restarting does not end its leases. */ +const VIEW_LINGER_MS = 30_000; + +/** + * Environments of envs.json through the envs plugin's SSH manager (its keys and known hosts; read at each connect, as + * it reopens with a reload). + */ +const sshEnvironments: EnvConnector = (self) => { + const managers = nativeOf(self, EnvManagers, "The SSH manager"); + return (name, config) => managers().manager.connection(name, config); +}; + +export default mcpServerFacet(VIEW_LINGER_MS, sshEnvironments); + +/** + * The facet, with views lingering `viewLingerMs` after their last reference, reaching environments through `connector` + * (tests shorten the one and replace the other). + */ +export function mcpServerFacet(viewLingerMs: number, connector: EnvConnector): Facet { + return defineFacet((host) => { + host.serve(McpManager, managerFactory(connector)); + host.serve(Mcp, view, { linger: viewLingerMs }); + }); +} + +/** + * The owner's configuration as last validly read, and why the last read failed (null: it did not). In memory, kept + * across a reload: a new reader seeing a save caught half-way keeps these servers. + */ +const Configured = defineTypedState("p2.mcp.config", { + schema: obj({ servers: unsafe<OwnerServer[]>(unknown()), errors: arr(str()), failure: union([str(), nul()]) }), + version: 1, + initial: () => ({ servers: [], errors: [], failure: null }), +}); + +/** The leases of session workers; kept across a reload of the plugin, so the new supervisor starts what they use. */ +const Leased = defineTypedState("p2.mcp.leases", { + schema: obj({ + leases: arr( + obj({ + session: str(), + location: LocationSchema, + project: union([ + obj({ kind: literal("none") }), + obj({ kind: literal("file"), text: str() }), + obj({ kind: literal("unreadable"), error: str() }), + ]), + generation: int(), + incarnation: str(), + }), + ), + }), + version: 1, + initial: (): { leases: Lease[] } => ({ leases: [] }), +}); + +/** The supervisor's instances as it last listed them; a reload's new supervisor starts with their tools. */ +const Listed = defineTypedState("p2.mcp.instances", { + schema: obj({ instances: arr(InstanceSchema) }), + version: 1, + initial: () => ({ instances: [] }), +}); + +/** The manager: its documents, the file and the supervisor as natives, and the routines that keep them in step. */ +function managerFactory(connector: EnvConnector): Factory<typeof McpManager> { + return (_params, self) => { + const server = self.get(Server, {}); + const envs = self.get(EnvService, {}); + const work = workEnvironments(self.get(Catalog, {}), self.get(ChildStore, {}), self); + const configured = self.state(Configured); + const leased = self.state(Leased); + const listed = self.state(Listed); + const placed = placements(); + const placement = () => placed(configured.value, leased.value.leases, envs.configs.value, work.value); + const connect = connector(self); + const file = self.open( + async () => { + const opened = new OwnerConfigFile(server.info.value.dataDir); + await opened.observer.start(); + return opened; + }, + (opened) => opened.observer.close(), + ); + const instances = self.open( + async () => { + // After a reload the leases are still here: the routine below starts what they use, with the tools listed + // before. Not here: natives open before derives compute. + const info = await clientInfo(server.info.value.app); + return new Instances(info, RECONNECT, listed.value.instances, connect); + }, + (opened) => opened.close(), + ); + self.onLive((ctx) => followNatives(file, instances, configured, listed, ctx)); + // The current configuration, leases, environments and places of work, then every change of them, start and close + // instances; methods that change leases reconcile at once too, so a call right after `setLocations` finds its + // instance. + const inputs = deriveTyped( + self, + [configured, leased, envs.configs, work], + ([config, { leases }, configs, at]) => ({ + config, + leases, + envs: configs, + work: at, + }), + ); + self.on(inputs, async ({ value }) => + instances.value.reconcile(placed(value.config, value.leases, value.envs, value.work)), + ); + const leases = deriveTyped(self, [inputs], ([value]) => + placed(value.config, value.leases, value.envs, value.work).leases.map(({ lease, servers, errors }) => ({ + session: lease.session, + location: lease.location, + instances: [...new Set(servers.values())], + errors: [...errors], + })), + ); + return { + instances: deriveTyped(self, [listed], ([value]) => value.instances), + leases, + errors: self.derive([configured], ([{ errors, failure }]) => + failure === null ? [...errors] : [...errors, `${failure}; keeping the last valid servers`], + ), + ...managerMethods(leased, () => instances.value, placement), + }; + }; +} + +/** The environments each session's conversations work on, as the catalog and the subagent store record them. */ +function workEnvironments( + catalog: Ref<typeof Catalog>, + store: Ref<typeof ChildStore>, + self: Self, +): Watchable<WorkEnvironments> { + return deriveTyped(self, [catalog.sessions, store.children], ([sessions, children]) => + Object.fromEntries(sessions.map((session) => [session.id, sessionEnvironments(session, children[session.id])])), + ); +} + +/** The environments of `session`'s conversations: its own, then its subagents' own locations' (archived ones too). */ +function sessionEnvironments(session: Session, children: readonly Child[] | undefined): (string | null)[] { + return [...new Set([session.env ?? null, ...(children ?? []).flatMap((child) => child.location?.env ?? [])])]; +} + +/** + * `place`, computed once per input: the routine, the leases' view and the methods share it. Where sessions work is + * compared by content: the catalog changes with every turn's activity, mostly without moving a session. + */ +function placements(): ( + config: OwnerConfig, + leases: readonly Lease[], + envs: Readonly<Record<string, EnvConfig>>, + work: WorkEnvironments, +) => Placement { + let last: + | { config: OwnerConfig; leases: readonly Lease[]; envs: object; work: string; placement: Placement } + | undefined; + return (config, leases, envs, work) => { + const at = JSON.stringify(work); + if (last?.config !== config || last.leases !== leases || last.envs !== envs || last.work !== at) + last = { config, leases, envs, work: at, placement: place(config, leases, envs, work) }; + return last.placement; + }; +} + +/** Copies what the file reads and the supervisor lists into the manager's documents until `ctx` ends. */ +async function followNatives( + file: Native<OwnerConfigFile>, + instances: Native<Instances>, + configured: HandleOf<typeof Configured>, + listed: HandleOf<typeof Listed>, + ctx: Ctx, +): Promise<void> { + // Changes heard while this routine stops would change state with an ended ctx. + const show = (): void => { + if (signalOf(ctx).aborted) return; + void listed + .change((draft) => { + draft.instances = instances.value.list(); + }, ctx) + .catch(() => {}); + }; + const stops = [ + file.value.subscribe((read) => { + if (signalOf(ctx).aborted) return; + void configured + .change((draft) => { + // An invalid read keeps the servers last read validly. + if (read.kind === "invalid") draft.failure = read.failure; + else if (read.kind === "valid") + Object.assign(draft, { + servers: [...read.config.servers], + errors: [...read.config.errors], + failure: null, + }); + }, ctx) + .catch(() => {}); + }), + instances.value.subscribe(show), + ]; + show(); + try { + await untilAborted(ctx); + } finally { + for (const stop of stops) stop(); + } +} + +/** The manager's methods: the lease rules applied on the leases' line, calls through the leased instance, restarts. */ +function managerMethods(leased: HandleOf<typeof Leased>, instances: () => Instances, placement: () => Placement) { + return { + setLocations: async ({ locations, projects = [], ...caller }: Caller & LocationsArgs, ctx: Ctx) => { + const places = locations.map((location) => ({ location, project: projectAt(location, projects) })); + let refused: string | undefined; + // Decided on the object's line: two generations' concurrent calls see each other's leases. + await leased.change((draft) => { + const result = setLocations(draft.leases, caller, places); + if (result.kind === "refused") refused = result.reason; + else draft.leases = result.leases; + }, ctx); + if (refused !== undefined) throw forbidden(refused); + instances().reconcile(placement()); + return null; + }, + releaseLeases: async ({ session, generation }: { session: string; generation: number }, ctx: Ctx) => { + await leased.change((draft) => { + draft.leases = releaseGeneration(draft.leases, session, generation); + }, ctx); + instances().reconcile(placement()); + return null; + }, + call: async ({ location, server: name, tool, args, ...caller }: Caller & CallArgs, ctx: Ctx) => { + const held = heldLease(leased.value.leases, caller, location); + if (held.kind === "stale") throw forbidden("Another worker of this session holds its MCP servers"); + if (held.kind === "none") throw new Error(`No MCP servers are leased at ${locationText(location)}`); + const id = placement() + .leases.find((placed) => placed.lease === held.lease) + ?.servers.get(name); + if (id === undefined) throw new Error(`MCP server "${name}" is not configured at ${locationText(location)}`); + return instances().call(id, tool, args, signalOf(ctx)); + }, + restart: async ({ instance }: { instance: string }) => { + instances().restart(instance); + return null; + }, + }; +} + +/** The project configuration the worker found at `location`: none, the file's text, or why it could not read it. */ +function projectAt(location: Location, projects: readonly ProjectFile[]): ProjectSource { + const file = projects.find((candidate) => sameLocation(candidate.location, location)); + if (file === undefined) return { kind: "none" }; + return "text" in file ? { kind: "file", text: file.text } : { kind: "unreadable", error: file.error }; +} + +/** The view: the owner's, a session worker's, nobody else's. */ +const view: Factory<typeof Mcp> = (_params, principal, self) => { + if (!isOwner(principal) && !isWorker(principal)) + throw forbidden("MCP servers are for the owner and session workers"); + const manager = self.get(McpManager, {}); + if (isOwner(principal)) + return { + variant: "owner", + members: { + instances: manager.instances, + errors: deriveTyped(self, [manager.errors, manager.leases], ([errors, leases]) => [ + ...errors, + ...leases.flatMap((lease) => lease.errors.map((error) => `session ${lease.session}: ${error}`)), + ]), + restart: (args, ctx) => manager.restart(args, ctx), + }, + }; + return { variant: "worker", members: workerView(principal, manager, self) }; +}; + +/** A session worker's view: its session's servers (without the owner's errors), its locations and calls. */ +function workerView(principal: WorkerPrincipal, manager: Ref<typeof McpManager>, self: Self) { + const { session, generation } = principal; + const current = currentWorker(self.get(Hosts, {}), principal); + // When the view ends (its worker's link went, and the linger passed), its leases go. A reload keeps them: the next + // attempt is the same worker. + self.open( + async () => null, + async (_value, reason, context) => { + if (reason.kind === "end") await manager.releaseLeases({ session, generation }, context); + }, + ); + return { + servers: deriveTyped(self, [manager.instances, manager.leases], ([instances, leases]) => { + const used = new Set(leases.filter((lease) => lease.session === session).flatMap((lease) => lease.instances)); + return instances.filter((instance) => used.has(instance.id)).map(({ error: _error, ...instance }) => instance); + }), + errors: deriveTyped(self, [manager.leases], ([leases]) => + leases.filter((lease) => lease.session === session).flatMap((lease) => lease.errors), + ), + setLocations: async (args: LocationsArgs, ctx: Ctx) => { + await current(ctx); + return manager.setLocations({ session, generation, incarnation: ctx.incarnation, ...args }, ctx); + }, + call: async ({ location, server, tool, args }: CallArgs, ctx: Ctx) => { + await current(ctx); + return manager.call({ session, generation, incarnation: ctx.incarnation, location, server, tool, args }, ctx); + }, + }; +} + +/** + * Refuses all but the session's running current generation, as the location tools do. A worker may ask before the + * server published its start: the check waits for that (the wait ends with the caller's call). + */ +function currentWorker( + hosts: Ref<typeof Hosts>, + { session, generation }: WorkerPrincipal, +): (ctx: Ctx) => Promise<void> { + const known = (state: WorkerState | undefined) => + state !== undefined && (state.generation > generation || (state.generation === generation && state.running)); + return async (ctx) => { + const published = hosts.workers.value[session]; + const state = known(published) + ? published + : (await until(hosts.workers, (workers) => known(workers[session]), ctx))[session]; + if (!state?.running || state.generation !== generation) throw forbidden("stale worker"); + }; +} + +/** p2 as MCP servers see it: its name and the app's version. */ +async function clientInfo(app: string): Promise<ClientInfo> { + // The app's own package.json: a version string. + const { version } = JSON.parse(await readFile(join(app, "package.json"), "utf8")) as { version: string }; + return { name: "p2", version }; +} diff --git a/apps/p2/plugins/mcp/tools.ts b/apps/p2/plugins/mcp/tools.ts new file mode 100644 index 000000000..cd1bd2144 --- /dev/null +++ b/apps/p2/plugins/mcp/tools.ts @@ -0,0 +1,114 @@ +// MCP tools as Durable tools: one deferred tool (found through `tool_search`) per tool of the session's servers, named +// as pi names them (`mcp__<server>__<tool>`), calling the server's instance where the calling conversation works. + +import type { JsonValue } from "@earendil-works/chord/json"; +import { + type ConversationId, + type DocumentReader, + defineExtension, + type Extension, + type ToolRegistration, +} from "@earendil-works/pi-durable"; +import { + type CallToolResult, + createMcpToolName, + limitMcpContent, + normalizeMcpInputSchema, + toLlmContent, +} from "@earendil-works/pi-mcp"; +import type { Context } from "@p2/app/v1"; +import { type Instance, MCP_EXTENSION, type McpTool } from "./contract.ts"; + +/** One tool of one server. */ +export type ServerTool = { readonly server: string; readonly tool: McpTool }; + +/** A call of `server`'s `tool` for conversation `conversationId` (its documents read through `read`). */ +export type McpCall = ( + request: { + readonly conversationId: ConversationId; + readonly read: DocumentReader; + readonly server: string; + readonly tool: string; + readonly args: Record<string, unknown>; + }, + context: Context, +) => Promise<CallToolResult>; + +/** The tools of `servers` by server and tool name; a server with instances at several locations lists a tool once. */ +export function serverTools(servers: readonly Instance[]): ServerTool[] { + const tools = new Map<string, ServerTool>(); + for (const instance of servers) + for (const tool of instance.tools) { + const key = JSON.stringify([instance.server, tool.name]); + if (!tools.has(key)) tools.set(key, { server: instance.server, tool }); + } + const order = (a: string, b: string) => (a < b ? -1 : a > b ? 1 : 0); + return [...tools.values()].sort((a, b) => order(a.server, b.server) || order(a.tool.name, b.tool.name)); +} + +/** + * The Durable extension offering `tools`, each calling through `call`. As in pi (and Codex), every tool whose name + * sanitizes to one another tool's also gets pi's hash suffix, so no tool's name depends on the order of the list + * and a new tool renames only those it collides with. + */ +export function createMcpExtension(tools: readonly ServerTool[], call: McpCall): Extension { + const plain = new Map<string, number>(); + for (const { server, tool } of tools) { + const name = createMcpToolName(server, tool.name); + plain.set(name, (plain.get(name) ?? 0) + 1); + } + const assigned = new Set<string>(); + return defineExtension({ + name: MCP_EXTENSION, + tools: tools.map((entry) => { + const name = createMcpToolName( + entry.server, + entry.tool.name, + (candidate) => (plain.get(candidate) ?? 0) > 1 || assigned.has(candidate), + ); + assigned.add(name); + return toolRegistration(name, entry, call); + }), + }); +} + +/** One MCP tool as a deferred Durable tool (not replay-safe: an interrupted call is not repeated). */ +function toolRegistration(name: string, { server, tool }: ServerTool, call: McpCall): ToolRegistration { + const title = tool.title ?? tool.annotations?.title; + return { + name, + description: tool.description?.trim() || title || `MCP tool ${tool.name} from server ${server}`, + parameters: normalizeMcpInputSchema(tool.inputSchema), + deferred: true, + ...(tool.outputSchema === undefined ? {} : { structuredOutputSchema: tool.outputSchema }), + async execute(args, api, context) { + // Durable checked `args` against `parameters`, an object schema; its type cannot follow a runtime schema. + const result = await call( + { + conversationId: api.conversationId, + read: api, + server, + tool: tool.name, + args: args as Record<string, unknown>, + }, + context, + ); + const content = toLlmContent(result); + if (result.isError && !content.some((block) => block.type === "text" && block.text.trim() !== "")) + content.push({ type: "text", text: `MCP tool ${server}/${tool.name} returned an error` }); + const limited = await limitMcpContent(content); + // Structured output only for a tool that declares its schema (Durable checks it), error results too. The + // server's JSON arrived as JSON; the type says `unknown` only because MCP allows any value. + const structured = + tool.outputSchema !== undefined && result.structuredContent !== undefined + ? { structuredOutput: result.structuredContent as JsonValue } + : {}; + return { + output: limited.content, + isError: result.isError === true, + details: { server, tool: tool.name }, + ...structured, + }; + }, + }; +} diff --git a/apps/p2/plugins/mcp/transports.ts b/apps/p2/plugins/mcp/transports.ts new file mode 100644 index 000000000..6d6a2603f --- /dev/null +++ b/apps/p2/plugins/mcp/transports.ts @@ -0,0 +1,94 @@ +// How an instance opens its MCP transport, by where it runs: a program on the p2 host (`StdioTransport`), a program +// on an environment through its pi-env daemon (`PipeTransport` over `openProcess`, through `cmd.exe` on Windows), or +// a Streamable HTTP server. + +import { posix, win32 } from "node:path"; +import { pathToFileURL } from "node:url"; +import { type Connection, nodePlatform, openProcess, RemoteError } from "@earendil-works/pi-env"; +import { type McpTransport, PipeTransport, StdioTransport, StreamableHttpTransport } from "@earendil-works/pi-mcp"; +import { expandHostVariables } from "./config.ts"; +import type { EnvConnections } from "./environments.ts"; +import type { Connect, RemoteConnect } from "./placement.ts"; +import { windowsCommand } from "./windows-command.ts"; + +/** A transport, not started yet, and the `file://` roots its client offers. */ +export type Opened = { readonly transport: McpTransport; readonly roots: readonly string[] }; + +/** How one instance opens transports, holding what they need meanwhile (an environment's connection) until `release`. */ +export type Transports = { open(): Promise<Opened>; release(): void }; + +/** The environment's daemon predates the `process` operation: retrying does not help, updating it does. */ +export class DaemonTooOldError extends Error {} + +/** The transports of an instance connecting as `connect`; an environment's connection is taken from `envs`. */ +export function transportsFor(connect: Connect, envs: EnvConnections): Transports { + switch (connect.kind) { + case "remote": { + const lease = envs.acquire(connect.environment, connect.config); + return { open: () => remoteTransport(connect, lease.connection), release: () => lease.release() }; + } + case "host": + return { + open: async () => ({ + transport: new StdioTransport({ + command: connect.command, + args: connect.args, + cwd: connect.cwd, + env: connect.hostVariables ? expandValues(connect.env) : { ...connect.env }, + stderr: "pipe", + }), + roots: connect.roots, + }), + release: () => {}, + }; + case "http": + return { + open: async () => ({ + transport: new StreamableHttpTransport({ url: connect.url, headers: expandValues(connect.headers) }), + roots: [], + }), + release: () => {}, + }; + case "unplaced": + return { + open: async () => { + throw new Error(connect.reason); + }, + release: () => {}, + }; + } +} + +/** + * A program on an environment: `~` is the remote home, relative paths are relative to the location's cwd there (else + * the home), in the remote system's path rules; on Windows, through `cmd.exe` unless it is an `.exe` or `.com`. + */ +async function remoteTransport(connect: RemoteConnect, connection: Connection): Promise<Opened> { + const info = await connection.info(); + const windows = nodePlatform(info).platform === "win32"; + const paths = windows ? win32 : posix; + const expandHome = (value: string): string => + value === "~" ? info.home : value.startsWith("~/") ? paths.join(info.home, value.slice(2)) : value; + const cwd = paths.resolve(connect.base ?? info.home, expandHome(connect.cwd)); + const command = expandHome(connect.command); + const args = connect.args.map(expandHome); + const launch = windows ? windowsCommand(command, args) : { argv: [command, ...args], env: {} }; + const env = { ...(connect.hostVariables ? expandValues(connect.env) : connect.env), ...launch.env }; + const transport = new PipeTransport({ + open: async ({ onStdout, onStderr }) => { + try { + return await openProcess(connection, { cwd, argv: launch.argv, env, onStdout, onStderr }); + } catch (error) { + if (error instanceof RemoteError && error.code === "not_supported") + throw new DaemonTooOldError(`the pi-env daemon on ${connect.environment} is too old; update it`); + throw error; + } + }, + }); + return { transport, roots: connect.base === null ? [] : [pathToFileURL(connect.base, { windows }).href] }; +} + +/** `values` with `${NAME}` from the p2 host's environment (the owner's configuration only). */ +function expandValues(values: Readonly<Record<string, string>>): Record<string, string> { + return Object.fromEntries(Object.entries(values).map(([key, value]) => [key, expandHostVariables(value)])); +} diff --git a/apps/p2/plugins/mcp/windows-command.ts b/apps/p2/plugins/mcp/windows-command.ts new file mode 100644 index 000000000..413f107cd --- /dev/null +++ b/apps/p2/plugins/mcp/windows-command.ts @@ -0,0 +1,49 @@ +// How a stdio server starts on a Windows environment: the pi-env daemon starts `.exe` and `.com` programs directly +// and refuses batch files (`npx` is `npx.cmd`), so every other command runs through `cmd.exe`, quoted as cross-spawn +// quotes it for Node on Windows. + +import { win32 } from "node:path"; + +/** The argv to start and the variables to add to the program's environment. */ +export type WindowsCommand = { readonly argv: readonly string[]; readonly env: Readonly<Record<string, string>> }; + +/** The variable holding the quoted command line `cmd.exe` runs. */ +const COMMAND_VARIABLE = "P2_MCP_COMMAND"; + +const EXECUTABLE = /\.(?:com|exe)$/i; + +const CMD_SHIM = /node_modules[\\/].bin[\\/][^\\/]+\.cmd$/i; + +/** + * `command` with `args` as the daemon starts it on Windows: directly for an `.exe` or `.com` path, else through + * `cmd.exe /d /s /c`, which finds `npx.cmd` for `npx` through `PATHEXT` as a shell would. + * + * The daemon quotes every argv element itself (libuv's rules) and has no verbatim mode, so a command line in an + * argument would reach `cmd.exe` with its quotes escaped as `\"`, which `cmd.exe` does not understand. The command + * line goes in an environment variable instead, and `cmd.exe` gets `%P2_MCP_COMMAND%`, which needs no quoting: it + * expands the variable before it parses quotes and `^` escapes, so it reads the line as cross-spawn's verbatim one. + */ +export function windowsCommand(command: string, args: readonly string[]): WindowsCommand { + if (EXECUTABLE.test(command)) return { argv: [command, ...args], env: {} }; + // A cmd-shim in node_modules\.bin passes its arguments through a second cmd.exe parse: their `^` escapes twice. + const quote = CMD_SHIM.test(command) ? (arg: string) => escapeCommand(escapeArgument(arg)) : escapeArgument; + const line = [escapeCommand(win32.normalize(command)), ...args.map(quote)].join(" "); + return { argv: ["cmd.exe", "/d", "/s", "/c", `%${COMMAND_VARIABLE}%`], env: { [COMMAND_VARIABLE]: line } }; +} + +/** cmd.exe's meta characters (cross-spawn's list, after robvanderwoude.com/escapechars.php). */ +const META = /([()\][%!^"`<>&|;, *?])/g; + +/** The program's name for cmd.exe: every meta character escaped with `^` (cross-spawn's `escapeCommand`). */ +function escapeCommand(command: string): string { + return command.replace(META, "^$1"); +} + +/** + * One argument for cmd.exe and then the program's own parsing (qntm.org/cmd, as cross-spawn): backslashes before a + * quote or the end doubled, quotes escaped, the whole quoted, then every meta character escaped with `^`. + */ +function escapeArgument(arg: string): string { + // The lookahead form keeps the regular expressions from backtracking on long runs of backslashes (cross-spawn #160). + return escapeCommand(`"${arg.replace(/(?=(\\+?)?)\1"/g, '$1$1\\"').replace(/(?=(\\+?)?)\1$/, "$1$1")}"`); +} diff --git a/apps/p2/plugins/mcp/worker.ts b/apps/p2/plugins/mcp/worker.ts new file mode 100644 index 000000000..3c3623050 --- /dev/null +++ b/apps/p2/plugins/mcp/worker.ts @@ -0,0 +1,218 @@ +// The MCP plugin's worker facet: `tool_search` (deferred tools, MCP's among them, are found through it) and the worker +// object holding the server's `Mcp` view. It sets the locations the session's conversations work at (the session's, +// active subagents' own), with the project `.pi/mcp.json` read at each through its environment, as its leases, again +// whenever they may have changed and whenever the view (re)activates, and offers the servers' tools as one Durable +// extension that follows the view's `servers`. No MCP process or credential is ever in a worker. + +import { setTimeout as sleep } from "node:timers/promises"; +import { type ConversationId, type Extension, type Harness, ROOT_CONVERSATION_ID } from "@earendil-works/pi-durable"; +import { createToolSearchExtension } from "@earendil-works/pi-durable/tool-search"; +import { + type Context, + type Ctx, + contribute, + defineFacet, + expectVariant, + isChordError, + type Location, + type Ref, + type StoreSelf, + sameLocation, + signalOf, + until, + untilAborted, + type Writable, +} from "@p2/app/v1"; +import { durableContext } from "@p2/app/v1/node"; +import { + CommitSignal, + type ExecutionEnvironments, + Extensions, + type HarnessParts, + harnessOf, + type WorkerParts, + workerOf, +} from "@p2/app/v1/worker"; +import { Subagents } from "../subagents/contract.worker.ts"; +import { type Instance, MCP_EXTENSION, Mcp, PROJECT_FILE_MAX_CHARS, type ProjectFile } from "./contract.ts"; +import { createMcpExtension, type McpCall, serverTools } from "./tools.ts"; + +/** How long to wait before setting the locations again after a failure. */ +const RETRY_MS = 2000; + +export default defineFacet((host, facet) => { + const search = createToolSearchExtension(); + facet.contribute(Extensions, { [search.name]: search }); + contribute(host, "p2.mcp-worker", mcpWorker); +}); + +/** The worker object: the view, the extension of the servers' tools, and the leases at the conversations' locations. */ +function mcpWorker(self: StoreSelf): void { + const harness = harnessOf(self); + const worker = workerOf(self); + const view = self.get(Mcp, {}); + const call: McpCall = async ({ conversationId, read, ...request }, context) => { + const location = await harness().environments.locate(conversationId, context, read); + return worker().root( + (ctx) => expectVariant(view, "worker").call({ location, ...request }, ctx), + context.abortSignal, + ); + }; + const extension = self.writable<Readonly<Record<string, Extension>>>({ + [MCP_EXTENSION]: createMcpExtension([], call), + }); + self.contribute(Extensions, extension); + self.onLive((ctx) => offerTools(view, extension, call, ctx)); + leaseLocations(self, view, harness, worker); +} + +/** Follows the view's `servers` until `ctx` ends: a new tool list (not a status change) makes a new extension. */ +async function offerTools( + view: Ref<typeof Mcp>, + extension: Writable<Readonly<Record<string, Extension>>>, + call: McpCall, + ctx: Ctx, +): Promise<void> { + let offered = "[]"; + const offer = (servers: readonly Instance[]): void => { + const tools = serverTools(servers); + const key = JSON.stringify(tools); + if (key === offered) return; + offered = key; + extension.set({ [MCP_EXTENSION]: createMcpExtension(tools, call) }); + }; + const watch = await expectVariant(view, "worker").servers.watch(ctx); + try { + offer(watch.value); + watch.start(async (change) => { + if (change.kind === "value") offer(change.value); + }); + await untilAborted(ctx); + } finally { + await watch.stop(); + } +} + +/** + * Sets the locations the conversations work at as the view's leases: once live, then after every subagent commit, + * move and re-activation of the view; a failure is retried, a refusal (a newer worker runs) ends it. + */ +function leaseLocations( + self: StoreSelf, + view: Ref<typeof Mcp>, + harness: () => HarnessParts, + worker: () => WorkerParts, +) { + // Subagents' own locations change through commits of their registry; the session's through moves. + const commits = self.open( + async () => + new CommitSignal(harness().durable, (publication) => + publication.changes.some( + (change) => change.type === "document" && change.record.kind === Subagents.definition.kind, + ), + ), + (signal) => signal.close(), + ); + // Counts re-activations of the view (a new view object, a reload): its first activation is the loop's first send. + const activated = self.writable(0); + let previous: string | undefined; + self.on(view.state, async ({ value }) => { + if (value.kind === "live" && previous !== undefined && previous !== "live") activated.set(activated.value + 1); + previous = value.kind; + }); + const activations = self.derive([activated], ([count]) => count); + self.onLive(async (ctx) => { + const signal = commits.value; + while (!signalOf(ctx).aborted) { + const seen = { commits: signal.version, moves: worker().location.version, views: activated.value }; + try { + const { environments, durable } = harness(); + const locations = await workLocations(durable, environments, durableContext(ctx)); + const projects = await projectFiles(locations, environments, durableContext(ctx)); + await expectVariant(view, "worker").setLocations({ locations, projects }, ctx); + } catch (error) { + if (signalOf(ctx).aborted) return; + if (isChordError(error, "forbidden")) { + console.error("p2 mcp: a newer worker of this session leases its MCP servers; this one stops:", error); + return; + } + console.error("p2 mcp: setting the session's MCP locations failed, retrying:", error); + await sleep(RETRY_MS, undefined, { signal: signalOf(ctx) }).catch(() => {}); + continue; + } + // A move or an activation while setting is not missed: its version is newer. + if (worker().location.version !== seen.moves || activated.value !== seen.views) continue; + await nextChange( + [ + (stop) => signal.next(seen.commits, stop), + (stop) => worker().location.next(stop), + (stop) => until(activations, (count) => count !== seen.views, durableContext(stop)), + ], + signalOf(ctx), + ); + } + }); +} + +/** Every distinct location the session's conversations work at: the main one's and active subagents'. */ +async function workLocations( + durable: Harness, + environments: ExecutionEnvironments, + context: Context, +): Promise<Location[]> { + const registry = await durable.snapshot(Subagents, ROOT_CONVERSATION_ID, context); + // Archived subagents rest until a send restores them (a commit); one-shot agents always follow the session. + const ids: ConversationId[] = [ + ROOT_CONVERSATION_ID, + ...Object.values(registry?.agents ?? {}) + .filter((agent) => !agent.archived) + .map((agent) => agent.conversationId), + ]; + const locations: Location[] = []; + for (const id of ids) { + const location = await environments.locate(id, context); + if (!locations.some((known) => sameLocation(known, location))) locations.push(location); + } + return locations; +} + +/** + * The project configurations at `locations`: `<cwd>/.pi/mcp.json` read through each location's environment (the + * manager checks them). A missing file is none; one too large or unreadable is reported as such. + */ +async function projectFiles( + locations: readonly Location[], + environments: ExecutionEnvironments, + context: Context, +): Promise<ProjectFile[]> { + const files: ProjectFile[] = []; + for (const location of locations) { + const env = await environments.at(location); + const path = await env.joinPath([location.cwd, ".pi", "mcp.json"], context); + const read = path.ok ? await env.readTextFile(path.value, context) : path; + if (read.ok) + files.push( + read.value.length <= PROJECT_FILE_MAX_CHARS + ? { location, text: read.value } + : { location, error: `larger than ${PROJECT_FILE_MAX_CHARS} characters` }, + ); + else if (read.error.code !== "not_found") files.push({ location, error: read.error.message.slice(0, 2000) }); + } + return files; +} + +/** Resolves after the first of `waits` (each given a signal that ends the others); an abort just ends the wait. */ +async function nextChange( + waits: readonly ((stop: AbortSignal) => Promise<unknown>)[], + aborted: AbortSignal, +): Promise<void> { + const done = new AbortController(); + const abort = (): void => done.abort(); + aborted.addEventListener("abort", abort, { once: true }); + try { + await Promise.race(waits.map((wait) => wait(done.signal))).catch(() => {}); + } finally { + done.abort(); + aborted.removeEventListener("abort", abort); + } +} diff --git a/apps/p2/plugins/models/fake.ts b/apps/p2/plugins/models/fake.ts index fec0f3727..3cdc01111 100644 --- a/apps/p2/plugins/models/fake.ts +++ b/apps/p2/plugins/models/fake.ts @@ -169,7 +169,8 @@ export function createFakeModels(options: CreateModelsOptions = {}) { const task = userText(transcript.messages.find((message) => message.role === "user")); if (schema !== undefined && !task.startsWith("json ") && !task.startsWith("slow")) return fauxAssistantMessage([fauxText(JSON.stringify(sampleOf(schema)))]); - const last = transcript.messages.at(-1); + // A section that changed during the run (tools loaded by `tool_search`) adds a system message after the result. + const last = transcript.messages.findLast((message) => message.role !== "system"); if (last?.role !== "toolResult") { const call = Object.hasOwn(SCRIPTED, text) ? SCRIPTED[text]?.() @@ -186,7 +187,9 @@ export function createFakeModels(options: CreateModelsOptions = {}) { ? fauxToolCall("subagent", JSON.parse(text.slice("subagent ".length))) : text.startsWith("team ") ? fauxToolCall("team", JSON.parse(text.slice("team ".length))) - : /^(history_(search|read|handoff)|location_(envs|dirs|move)) /.test(text) + : /^(history_(search|read|handoff)|location_(envs|dirs|move)|tool_search|mcp__\w+) /.test( + text, + ) ? fauxToolCall( text.slice(0, text.indexOf(" ")), JSON.parse(text.slice(text.indexOf(" ") + 1)), diff --git a/apps/p2/src/api/v1/index.ts b/apps/p2/src/api/v1/index.ts index a3f8dfbfa..08ab7618f 100644 --- a/apps/p2/src/api/v1/index.ts +++ b/apps/p2/src/api/v1/index.ts @@ -20,6 +20,7 @@ export { method, type Native, orderedEntries, + PluginError, type RefState, type Registry, type RegistryValue, diff --git a/apps/p2/src/hosts/plugins.ts b/apps/p2/src/hosts/plugins.ts index 8df6a8b90..757c5645a 100644 --- a/apps/p2/src/hosts/plugins.ts +++ b/apps/p2/src/hosts/plugins.ts @@ -22,8 +22,11 @@ import * as piDurable from "@earendil-works/pi-durable"; import * as durableCodemode from "@earendil-works/pi-durable/codemode"; import * as durableEnv from "@earendil-works/pi-durable/env"; import * as durableEnvNode from "@earendil-works/pi-durable/env/node"; +import * as durableToolSearch from "@earendil-works/pi-durable/tool-search"; import * as durableTools from "@earendil-works/pi-durable/tools"; import * as piEnv from "@earendil-works/pi-env"; +import * as piMcp from "@earendil-works/pi-mcp"; +import * as piMcpOauth from "@earendil-works/pi-mcp/oauth"; import * as piSchema from "@earendil-works/pi-schema"; import * as schemaFormats from "@earendil-works/pi-schema/formats"; import * as piTui from "@earendil-works/pi-tui"; @@ -65,8 +68,11 @@ const SHARED = { "@earendil-works/pi-durable/codemode": durableCodemode, "@earendil-works/pi-durable/env": durableEnv, "@earendil-works/pi-durable/env/node": durableEnvNode, + "@earendil-works/pi-durable/tool-search": durableToolSearch, "@earendil-works/pi-durable/tools": durableTools, "@earendil-works/pi-env": piEnv, + "@earendil-works/pi-mcp": piMcp, + "@earendil-works/pi-mcp/oauth": piMcpOauth, "@earendil-works/pi-schema": piSchema, "@earendil-works/pi-schema/formats": schemaFormats, ws, diff --git a/apps/p2/src/runtime/chord.ts b/apps/p2/src/runtime/chord.ts index 6f71384d0..237a7bcf3 100644 --- a/apps/p2/src/runtime/chord.ts +++ b/apps/p2/src/runtime/chord.ts @@ -197,14 +197,17 @@ export function newerPeerProtocol(error: { readonly kind: string; readonly error /** A plain listener source outside Chord (a watcher's events): `subscribe` returns its stop. */ export type Subscribable<T> = { subscribe(listener: (value: T) => void): () => void }; -/** Resolves once `source` holds a value `done` accepts (the current one included); aborting `context` rejects. */ -export async function until<T>(source: Watchable<T>, done: (value: T) => boolean, context: Context): Promise<void> { +/** + * Resolves with the first value of `source` that `done` accepts (the current one included); aborting `context` + * rejects. + */ +export async function until<T>(source: Watchable<T>, done: (value: T) => boolean, context: Context): Promise<T> { const watch = await source.watch(context); try { - if (done(watch.value)) return; - await new Promise<void>((resolve, reject) => { + if (done(watch.value)) return watch.value; + return await new Promise<T>((resolve, reject) => { watch.start(async (call) => { - if (call.kind === "value" && done(call.value)) resolve(); + if (call.kind === "value" && done(call.value)) resolve(call.value); }); void watch.closed.then((end) => reject(end.kind === "failed" ? end.error : new Error(`watch ${end.kind}`))); }); diff --git a/apps/p2/test/fixtures/mcp-server.mjs b/apps/p2/test/fixtures/mcp-server.mjs new file mode 100644 index 000000000..a733a3032 --- /dev/null +++ b/apps/p2/test/fixtures/mcp-server.mjs @@ -0,0 +1,64 @@ +// A stdio MCP server for p2's MCP tests, in the style of pi-mcp's test/fixtures/stdio-server.mjs. Tools: `echo` +// ({ text }), `whoami` (its pid, its parent's, cwd, the client's roots and its `MCP_FIXTURE_*` variables), `crash` +// (answers, then exits), `grow` (adds the tool `extra` and announces the change). `MCP_FIXTURE_LOG`: a file every start +// appends its pid to; `MCP_FIXTURE_HANG`: never answers `initialize`. +import { appendFileSync } from "node:fs"; +import { createInterface } from "node:readline"; + +if (process.env.MCP_FIXTURE_LOG) appendFileSync(process.env.MCP_FIXTURE_LOG, `${process.pid}\n`); +const send = (message) => process.stdout.write(`${JSON.stringify({ jsonrpc: "2.0", ...message })}\n`); +const tools = [ + { + name: "echo", + description: "Echoes the text back", + inputSchema: { type: "object", properties: { text: { type: "string" } } }, + }, + { name: "whoami", description: "Reports the server process", inputSchema: { type: "object" } }, + { name: "crash", description: "Exits the server", inputSchema: { type: "object" } }, + { name: "grow", description: "Adds a tool", inputSchema: { type: "object" } }, +]; +let roots = null; +const lines = createInterface({ input: process.stdin, crlfDelay: Infinity }); +for await (const line of lines) { + const message = JSON.parse(line); + if (message.method === "notifications/initialized") { + send({ id: "roots", method: "roots/list" }); + continue; + } + if (message.id === "roots") { + roots = message.result?.roots ?? null; + continue; + } + if (!("id" in message)) continue; + if (message.method === "initialize") { + if (process.env.MCP_FIXTURE_HANG) continue; + send({ + id: message.id, + result: { + protocolVersion: "2025-06-18", + capabilities: { tools: { listChanged: true } }, + serverInfo: { name: "p2-fixture", version: "1.0.0" }, + }, + }); + } else if (message.method === "tools/list") send({ id: message.id, result: { tools } }); + else if (message.method === "tools/call") { + const { name, arguments: args } = message.params; + if (name === "echo") + send({ id: message.id, result: { content: [{ type: "text", text: `echo: ${args?.text}` }] } }); + else if (name === "whoami") { + const vars = Object.fromEntries(Object.entries(process.env).filter(([key]) => key.startsWith("MCP_FIXTURE_"))); + const text = JSON.stringify({ pid: process.pid, ppid: process.ppid, cwd: process.cwd(), roots, vars }); + send({ id: message.id, result: { content: [{ type: "text", text }] } }); + } else if (name === "crash") { + send({ id: message.id, result: { content: [{ type: "text", text: "bye" }] } }); + console.error("fixture crashing on purpose"); + setTimeout(() => process.exit(1), 20); + } else if (name === "grow") { + if (!tools.some((tool) => tool.name === "extra")) + tools.push({ name: "extra", description: "Added later", inputSchema: { type: "object" } }); + send({ id: message.id, result: { content: [{ type: "text", text: "grown" }] } }); + send({ method: "notifications/tools/list_changed" }); + } else send({ id: message.id, result: { content: [{ type: "text", text: `no tool ${name}` }], isError: true } }); + } else if (message.method === "ping") send({ id: message.id, result: {} }); + else send({ id: message.id, error: { code: -32601, message: "not found" } }); +} diff --git a/apps/p2/test/p2/mcp-environments.test.ts b/apps/p2/test/p2/mcp-environments.test.ts new file mode 100644 index 000000000..287b133be --- /dev/null +++ b/apps/p2/test/p2/mcp-environments.test.ts @@ -0,0 +1,55 @@ +// The supervisor's pi-env connections (plugins/mcp/environments.ts): one per environment and configuration revision, +// shared by its users, closed with the last one; an edited environment is another connection. Connections here never +// start (a command that does not exist): a closed one refuses at once with "Connection closed", an open one tries to +// start and fails to spawn. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { Connection } from "@earendil-works/pi-env"; +import type { EnvConfig } from "../../plugins/envs/config.ts"; +import { EnvConnections } from "../../plugins/mcp/environments.ts"; + +const GPU: EnvConfig = { type: "ssh", host: "gpu.example" }; +const EDITED: EnvConfig = { type: "ssh", host: "gpu.example", port: 2222 }; + +/** Whether `connection` was closed (it refuses at once) rather than open (it tries to start the daemon). */ +const isClosed = async (connection: Connection): Promise<boolean> => + connection.info().then( + () => false, + (error: Error) => error.message === "Connection closed", + ); + +test("users of one environment share its connection; the last release closes it", async () => { + const opened: string[] = []; + const envs = new EnvConnections((name) => { + opened.push(name); + return new Connection({ command: ["/nonexistent/pi-env"] }); + }); + const first = envs.acquire("gpu", GPU); + const second = envs.acquire("gpu", GPU); + assert.equal(second.connection, first.connection); + const other = envs.acquire("box", GPU); + assert.notEqual(other.connection, first.connection); + assert.deepEqual(opened, ["gpu", "box"]); + first.release(); + assert.equal(await isClosed(second.connection), false); + second.release(); + assert.equal(await isClosed(second.connection), true); + assert.equal(await isClosed(other.connection), false); + other.release(); + // A new user after the last one opens a new connection. + const again = envs.acquire("gpu", GPU); + assert.notEqual(again.connection, first.connection); + assert.deepEqual(opened, ["gpu", "box", "gpu"]); + again.release(); +}); + +test("an edited environment is another connection; the old one stays for its users until they release it", async () => { + const envs = new EnvConnections(() => new Connection({ command: ["/nonexistent/pi-env"] })); + const before = envs.acquire("gpu", GPU); + const after = envs.acquire("gpu", EDITED); + assert.notEqual(after.connection, before.connection); + before.release(); + assert.equal(await isClosed(before.connection), true); + assert.equal(await isClosed(after.connection), false); + after.release(); +}); diff --git a/apps/p2/test/p2/mcp-instances.test.ts b/apps/p2/test/p2/mcp-instances.test.ts new file mode 100644 index 000000000..200613976 --- /dev/null +++ b/apps/p2/test/p2/mcp-instances.test.ts @@ -0,0 +1,139 @@ +// The MCP supervisor (plugins/mcp/instances.ts) on its own, with the stdio fixture (test/fixtures/mcp-server.mjs) and a +// short backoff: reconnects after a crash with the tools kept and the stderr only in the owner's error, tool-list +// changes, a close or restart while connecting, and the tools a reload's new supervisor starts with. +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, realpathSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { parseOwnerConfig } from "../../plugins/mcp/config.ts"; +import type { Instance } from "../../plugins/mcp/contract.ts"; +import { Instances } from "../../plugins/mcp/instances.ts"; +import { place } from "../../plugins/mcp/placement.ts"; +import { poll } from "./helpers.ts"; + +const FIXTURE = fileURLToPath(new URL("../fixtures/mcp-server.mjs", import.meta.url)); +const BACKOFF = { firstMs: 20, maxMs: 200, stableMs: 60_000 }; +const CLIENT = { name: "p2-test", version: "1" }; +const SIGNAL = new AbortController().signal; +/** These instances run on the p2 host only. */ +const NO_ENVIRONMENTS = (): never => { + throw new Error("no environments in this test"); +}; + +/** A supervisor running the fixture once for session s1 at a temporary directory; `env` added to its environment. */ +function supervisor(t: { after(fn: () => unknown): void }, env: Record<string, string> = {}, seed: Instance[] = []) { + const root = realpathSync(mkdtempSync(join(tmpdir(), "p2-mcp-instances-"))); + const log = join(root, "starts.log"); + const config = parseOwnerConfig( + JSON.stringify({ + mcpServers: { files: { command: process.execPath, args: [FIXTURE], env: { MCP_FIXTURE_LOG: log, ...env } } }, + }), + ); + const placement = place( + config, + [ + { + session: "s1", + location: { env: null, cwd: root }, + project: { kind: "none" }, + generation: 1, + incarnation: "i", + }, + ], + {}, + { s1: [null] }, + ); + const instances = new Instances(CLIENT, BACKOFF, seed, NO_ENVIRONMENTS); + t.after(async () => { + await instances.close(); + rmSync(root, { recursive: true, force: true }); + }); + const pids = () => (existsSync(log) ? readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map(Number) : []); + return { instances, placement, pids, id: [...placement.instances.keys()][0]! }; +} + +/** Resolves with the instances once `predicate` holds for them. */ +function when(instances: Instances, predicate: (list: Instance[]) => boolean): Promise<Instance[]> { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + stop(); + reject(new Error(`timed out; last: ${JSON.stringify(instances.list()).slice(0, 400)}`)); + }, 10_000); + const check = () => { + const list = instances.list(); + if (!predicate(list)) return; + clearTimeout(timer); + stop(); + resolve(list); + }; + const stop = instances.subscribe(check); + check(); + }); +} + +const alive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; +const exited = (pid: number) => poll(() => !alive(pid), 10_000, `process ${pid} to exit`); + +test("a crashed server reconnects after the backoff with its tools kept; its stderr is the owner's only", async (t) => { + const { instances, placement, pids, id } = supervisor(t); + instances.reconcile(placement); + await when(instances, (list) => list[0]?.status === "connected"); + await instances.call(id, "crash", {}, SIGNAL); + const [down] = await when(instances, (list) => list[0]?.status !== "connected"); + assert.equal(down!.tools.length, 4); + await assert.rejects(instances.call(id, "echo", {}, SIGNAL), (error: Error) => { + assert.equal(error.message, 'MCP server "files" is not connected: the connection was lost'); + return true; + }); + assert.match(down!.error ?? "", /stderr: fixture crashing on purpose/); + await exited(pids()[0]!); + const [back] = await when(instances, (list) => list[0]?.status === "connected"); + assert.equal(back!.tools.length, 4); + assert.equal(back!.error, undefined); + assert.equal(pids().length, 2); + // A tool-list change refreshes the tools. + await instances.call(id, "grow", {}, SIGNAL); + await when(instances, (list) => list[0]!.tools.some((tool) => tool.name === "extra")); +}); + +test("close and restart end a connect under way", async (t) => { + const { instances, placement, pids, id } = supervisor(t, { MCP_FIXTURE_HANG: "1" }); + instances.reconcile(placement); + await poll(() => pids().length === 1, 10_000, "the server to start"); + assert.equal(instances.list()[0]!.status, "starting"); + instances.restart(id); + await exited(pids()[0]!); + await poll(() => pids().length === 2, 10_000, "the server to start again"); + await instances.close(); + await exited(pids()[1]!); + assert.deepEqual(instances.list(), []); +}); + +test("a new supervisor lists the tools its predecessor listed until the server answers", async (t) => { + const first = supervisor(t, { MCP_FIXTURE_HANG: "1" }); + const seed: Instance = { + id: first.id, + server: "files", + source: "owner", + location: null, + runsOn: "host", + status: "connected", + tools: [{ name: "echo", inputSchema: { type: "object" } }], + }; + const instances = new Instances(CLIENT, BACKOFF, [seed], NO_ENVIRONMENTS); + t.after(() => instances.close()); + instances.reconcile(first.placement); + assert.deepEqual( + instances.list().map((instance) => [instance.status, instance.tools.map((tool) => tool.name)]), + [["starting", ["echo"]]], + ); +}); diff --git a/apps/p2/test/p2/mcp-leases.test.ts b/apps/p2/test/p2/mcp-leases.test.ts new file mode 100644 index 000000000..8e662b63f --- /dev/null +++ b/apps/p2/test/p2/mcp-leases.test.ts @@ -0,0 +1,89 @@ +// The MCP lease rules (plugins/mcp/leases.ts): a generation sets exactly its locations, a newer generation takes the +// session over and refuses an older one, calls go through the setting incarnation's lease, and a view's end releases +// only its own generation's leases; each lease keeps the project configuration sent with its location. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import type { Location } from "@p2/app/v1"; +import { heldLease, type Lease, releaseGeneration, setLocations } from "../../plugins/mcp/leases.ts"; + +const A: Location = { env: null, cwd: "/a" }; +const B: Location = { env: null, cwd: "/b" }; +const GPU: Location = { env: "gpu", cwd: "/a" }; +const NONE = { kind: "none" } as const; +const gen = (generation: number, incarnation = `i${generation}`) => ({ session: "s1", generation, incarnation }); +const set = (leases: readonly Lease[], caller: ReturnType<typeof gen>, locations: Location[]) => { + const result = setLocations( + leases, + caller, + locations.map((location) => ({ location, project: NONE })), + ); + assert.equal(result.kind, "set"); + return result.kind === "set" ? result.leases : []; +}; + +test("a generation leases exactly its locations; leaving one out ends its lease", () => { + const other: Lease = { session: "s2", location: A, project: NONE, generation: 1, incarnation: "x" }; + const first = set([other], gen(1), [A, B, A]); + assert.deepEqual( + first.map((lease) => [lease.session, lease.location.cwd]), + [ + ["s2", "/a"], + ["s1", "/a"], + ["s1", "/b"], + ], + ); + // The same location on another environment is another lease. + assert.equal(set(first, gen(1), [A, GPU]).length, 3); + const shrunk = set(first, gen(1), [B]); + assert.deepEqual( + shrunk.map((lease) => [lease.session, lease.location.cwd]), + [ + ["s2", "/a"], + ["s1", "/b"], + ], + ); + assert.deepEqual(set(shrunk, gen(1), []), [other]); +}); + +test("a newer generation takes the session over; an older one is refused", () => { + const older = set([], gen(1), [A, B]); + const newer = set(older, gen(2), [A]); + assert.deepEqual( + newer.map((lease) => [lease.location.cwd, lease.generation]), + [["/a", 2]], + ); + assert.deepEqual(setLocations(newer, gen(1), [{ location: A, project: NONE }]), { + kind: "refused", + reason: "A newer worker of this session holds its MCP servers", + }); + // Another incarnation of the same generation takes it over too (its calls then refuse the first one's). + const again = set(newer, gen(2, "other"), [A]); + assert.equal(heldLease(again, gen(2, "other"), A).kind, "held"); + assert.equal(heldLease(again, gen(2), A).kind, "stale"); + assert.equal(heldLease(again, gen(2, "other"), B).kind, "none"); +}); + +test("a view's end releases only its generation's leases", () => { + const leases = set([{ session: "s2", location: A, project: NONE, generation: 1, incarnation: "x" }], gen(2), [A]); + assert.deepEqual(releaseGeneration(leases, "s1", 1), leases); + assert.deepEqual( + releaseGeneration(leases, "s1", 2).map((lease) => lease.session), + ["s2"], + ); +}); + +test("a lease keeps the project configuration sent with its location; of a location given twice, the first counts", () => { + const file = { kind: "file", text: "{}" } as const; + const result = setLocations([], gen(1), [ + { location: A, project: file }, + { location: A, project: NONE }, + { location: B, project: { kind: "unreadable", error: "denied" } }, + ]); + assert.deepEqual( + result.kind === "set" ? result.leases.map((lease) => [lease.location.cwd, lease.project.kind]) : [], + [ + ["/a", "file"], + ["/b", "unreadable"], + ], + ); +}); diff --git a/apps/p2/test/p2/mcp-project.test.ts b/apps/p2/test/p2/mcp-project.test.ts new file mode 100644 index 000000000..24292e7b8 --- /dev/null +++ b/apps/p2/test/p2/mcp-project.test.ts @@ -0,0 +1,93 @@ +// A project's `.pi/mcp.json` checked against the owner's servers (plugins/mcp/project.ts): what it may add (stdio +// servers at its location), what it may change (`enabled` of the owner's servers), and each rule that refuses the rest. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { parseOwnerConfig } from "../../plugins/mcp/config.ts"; +import { parseProjectConfig } from "../../plugins/mcp/project.ts"; + +const OWNER = parseOwnerConfig( + JSON.stringify({ + mcpServers: { + files: { command: "files-server" }, + off: { command: "off-server", enabled: false }, + "my-docs": { url: "https://example.com/mcp" }, + }, + }), +); + +const project = (servers: Record<string, unknown>) => + parseProjectConfig(JSON.stringify({ mcpServers: servers }), OWNER); + +test("a project adds stdio servers and sets enabled of the owner's; host variables stay as written", () => { + const config = project({ + lint: { command: "lint-server", args: ["--fix"], env: { TOKEN: `\${SECRET}` } }, + later: { command: "later-server", enabled: false }, + files: { enabled: false }, + off: { enabled: true }, + "my-docs": {}, + }); + assert.deepEqual(config.errors, []); + assert.deepEqual( + config.servers.map((server) => [server.name, server.config.command, server.config.args, server.config.env]), + [["lint", "lint-server", ["--fix"], { TOKEN: `\${SECRET}` }]], + ); + // An entry without `enabled` is an override that changes nothing. + assert.deepEqual( + [...config.enabled], + [ + ["files", false], + ["off", true], + ], + ); +}); + +test("each rule refuses what it should, naming the server and why; the other entries still count", () => { + const config = project({ + remote: { url: "https://example.com/mcp" }, + placed: { command: "x", runOn: "host" }, + everywhere: { command: "x", shared: true }, + signed: { command: "x", oauth: { clientId: "c" } }, + provided: { command: "x", auth: { provider: "radius" } }, + files: { command: "other-files" }, + off: { enabled: true, args: ["x"] }, + "my-docs": { enabled: "yes" }, + ghost: { enabled: true }, + my_docs: { command: "x" }, + "bad name!": { command: "x" }, + kept: { command: "kept-server" }, + }); + assert.deepEqual( + config.servers.map((server) => server.name), + ["kept"], + ); + const errors = config.errors.join("\n"); + assert.equal(config.errors.length, 11, errors); + for (const error of config.errors) assert.match(error, /^\.pi\/mcp\.json: /); + assert.match(errors, /server "remote": HTTP servers are for the owner's mcp\.json/); + assert.match(errors, /server "placed": "runOn" is for the owner's mcp\.json/); + assert.match(errors, /server "everywhere": "shared" is for the owner's mcp\.json/); + assert.match(errors, /server "signed": "oauth" is for the owner's mcp\.json/); + assert.match(errors, /server "provided": "auth" is for the owner's mcp\.json/); + assert.match(errors, /server "files": the owner's mcp\.json defines it/); + assert.match(errors, /server "off": an override can only set "enabled"/); + assert.match(errors, /server "my-docs": enabled must be a boolean/); + assert.match(errors, /server "ghost": needs "command", or a server of the owner's mcp\.json to override/); + assert.match(errors, /server "my_docs": its tools would be named like those of "my-docs"/); + assert.match(errors, /invalid server name/); + assert.deepEqual([...config.enabled], []); +}); + +test("a file that is no configuration is one error, and nothing else", () => { + for (const [text, why] of [ + ["{", /Invalid \.pi\/mcp\.json syntax/], + ["[]", /\.pi\/mcp\.json must be an object/], + ['{ "mcpServers": [] }', /"mcpServers" must be an object/], + ] as const) { + const config = parseProjectConfig(text, OWNER); + assert.deepEqual([config.servers, [...config.enabled]], [[], []]); + assert.equal(config.errors.length, 1); + assert.match(config.errors[0]!, why); + } + // Comments are allowed, as in the owner's file. + assert.deepEqual(parseProjectConfig('{ // none\n "mcpServers": {} }', OWNER).errors, []); +}); diff --git a/apps/p2/test/p2/mcp-tools.test.ts b/apps/p2/test/p2/mcp-tools.test.ts new file mode 100644 index 000000000..7705fa631 --- /dev/null +++ b/apps/p2/test/p2/mcp-tools.test.ts @@ -0,0 +1,104 @@ +// MCP tools as Durable tools (plugins/mcp/tools.ts): pi's names, with the hash suffix for every colliding name whatever +// the order, structured output only for tools with an output schema (error results included), and error results with +// text. +import assert from "node:assert/strict"; +import { test } from "node:test"; +import type { ToolExecutionApi, ToolRegistration } from "@earendil-works/pi-durable"; +import type { CallToolResult } from "@earendil-works/pi-mcp"; +import { BACKGROUND_CONTEXT } from "@p2/app/v1"; +import type { McpTool } from "../../plugins/mcp/contract.ts"; +import { createMcpExtension, type McpCall, type ServerTool } from "../../plugins/mcp/tools.ts"; + +const tool = (name: string, extra: Partial<McpTool> = {}): McpTool => ({ + name, + inputSchema: { type: "object" }, + ...extra, +}); +const names = (tools: readonly ServerTool[]) => + (createMcpExtension(tools, async () => ({ content: [] })).tools ?? []).map((registration) => registration.name); + +test("tool names: pi's, and every tool of a colliding name gets the hash suffix whatever the order", () => { + const plain = [ + { server: "docs", tool: tool("search") }, + { server: "docs", tool: tool("a-b") }, + { server: "docs", tool: tool("a_b") }, + ]; + const [search, dash, underscore] = names(plain); + assert.equal(search, "mcp__docs__search"); + assert.match(dash!, /^mcp__docs__a_b_[0-9a-f]{8}$/); + assert.match(underscore!, /^mcp__docs__a_b_[0-9a-f]{8}$/); + assert.notEqual(dash, underscore); + // The other order gives each tool the same name. + assert.deepEqual(names([...plain].reverse()), [underscore, dash, search]); + // A new tool that collides with nothing renames nothing; one that collides renames only its rival. + assert.deepEqual(names([...plain, { server: "docs", tool: tool("fetch") }]), [ + search, + dash, + underscore, + "mcp__docs__fetch", + ]); + assert.equal(names([{ server: "docs", tool: tool("x-y") }])[0], "mcp__docs__x_y"); + const rivals = names([ + { server: "docs", tool: tool("search") }, + { server: "docs", tool: tool("x-y") }, + { server: "docs", tool: tool("x.y") }, + ]); + assert.equal(rivals[0], "mcp__docs__search"); + assert.match(rivals[1]!, /^mcp__docs__x_y_[0-9a-f]{8}$/); + assert.match(rivals[2]!, /^mcp__docs__x_y_[0-9a-f]{8}$/); +}); + +test("tool names: a collision across servers suffixes both tools too", () => { + const [first, second] = names([ + { server: "a", tool: tool("b__c") }, + { server: "a__b", tool: tool("c") }, + ]); + assert.match(first!, /^mcp__a__b__c_[0-9a-f]{8}$/); + assert.match(second!, /^mcp__a__b__c_[0-9a-f]{8}$/); + assert.notEqual(first, second); + assert.equal(names([{ server: "a", tool: tool("b__c") }])[0], "mcp__a__b__c"); +}); + +/** The registration of `mcpTool` calling a server that answers `result`, and what it was called with. */ +function registration(mcpTool: McpTool, result: CallToolResult) { + const calls: Parameters<McpCall>[0][] = []; + const call: McpCall = async (request) => { + calls.push(request); + return result; + }; + const [registered] = createMcpExtension([{ server: "docs", tool: mcpTool }], call).tools as ToolRegistration[]; + // The tool reads only the conversation ID and the reader from the API. + const api = { conversationId: 1 } as unknown as ToolExecutionApi; + return { registered: registered!, calls, run: () => registered!.execute({ q: "x" }, api, BACKGROUND_CONTEXT) }; +} + +test("results: structured output only with an output schema, error results included; errors always have text", async () => { + const schema = { type: "object", properties: { hits: { type: "number" } } }; + const structured = { content: [{ type: "text" as const, text: "2 hits" }], structuredContent: { hits: 2 } }; + const withSchema = registration(tool("search", { outputSchema: schema }), structured); + assert.deepEqual(withSchema.registered.structuredOutputSchema, schema); + assert.equal(withSchema.registered.deferred, true); + assert.deepEqual(await withSchema.run(), { + output: [{ type: "text", text: "2 hits" }], + isError: false, + details: { server: "docs", tool: "search" }, + structuredOutput: { hits: 2 }, + }); + assert.deepEqual(withSchema.calls[0]?.args, { q: "x" }); + assert.equal(withSchema.calls[0]?.tool, "search"); + const withoutSchema = registration(tool("search"), structured); + assert.equal(withoutSchema.registered.structuredOutputSchema, undefined); + assert.equal("structuredOutput" in (await withoutSchema.run()), false); + const failed = registration(tool("search", { outputSchema: schema }), { + content: [], + structuredContent: { hits: 0 }, + isError: true, + }); + const error = await failed.run(); + assert.equal(error.isError, true); + assert.deepEqual(error.structuredOutput, { hits: 0 }); + // Without content the structured content becomes the text; without either, an error still says so. + assert.match(JSON.stringify(error.output), /hits/); + const bare = await registration(tool("search"), { content: [], isError: true }).run(); + assert.deepEqual(bare.output, [{ type: "text", text: "MCP tool docs/search returned an error" }]); +}); diff --git a/apps/p2/test/p2/mcp-windows-command.test.ts b/apps/p2/test/p2/mcp-windows-command.test.ts new file mode 100644 index 000000000..133d334d6 --- /dev/null +++ b/apps/p2/test/p2/mcp-windows-command.test.ts @@ -0,0 +1,55 @@ +// How a stdio MCP server starts on a Windows environment (plugins/mcp/windows-command.ts): `.exe` and `.com` directly, +// everything else through `cmd.exe` with the command line in a variable, quoted as cross-spawn quotes it (the expected +// lines below are cross-spawn's `lib/util/escape.js` output for the same values). +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { windowsCommand } from "../../plugins/mcp/windows-command.ts"; + +const CMD = ["cmd.exe", "/d", "/s", "/c", "%P2_MCP_COMMAND%"]; + +test("an .exe or .com program starts directly, its arguments as they are", () => { + assert.deepEqual(windowsCommand("C:\\tools\\server.exe", ["a b", '"q"']), { + argv: ["C:\\tools\\server.exe", "a b", '"q"'], + env: {}, + }); + assert.deepEqual(windowsCommand("C:\\TOOLS\\SERVER.COM", []), { argv: ["C:\\TOOLS\\SERVER.COM"], env: {} }); +}); + +test("any other command runs through cmd.exe, the line in P2_MCP_COMMAND quoted as cross-spawn quotes it", () => { + const { argv, env } = windowsCommand("npx", [ + "plain", + "a b", + 'say "hi"', + "C:\\dir\\", + 'tail\\"q', + "100%", + "a&b|c", + "^", + ]); + assert.deepEqual(argv, CMD); + assert.equal( + env.P2_MCP_COMMAND, + [ + "npx", + '^"plain^"', + '^"a^ b^"', + '^"say^ \\^"hi\\^"^"', + '^"C:\\dir\\\\^"', + '^"tail\\\\\\^"q^"', + '^"100^%^"', + '^"a^&b^|c^"', + '^"^^^"', + ].join(" "), + ); +}); + +test("the program's path is normalized and its meta characters escaped", () => { + const { env } = windowsCommand("C:/Program Files (x86)/tool.cmd", []); + assert.equal(env.P2_MCP_COMMAND, "C:\\Program^ Files^ ^(x86^)\\tool.cmd"); +}); + +test("a cmd-shim in node_modules\\.bin gets its arguments escaped twice (it parses them through cmd.exe again)", () => { + const { argv, env } = windowsCommand("C:\\work\\node_modules\\.bin\\server.cmd", ["a b", "100%"]); + assert.deepEqual(argv, CMD); + assert.equal(env.P2_MCP_COMMAND, 'C:\\work\\node_modules\\.bin\\server.cmd ^^^"a^^^ b^^^" ^^^"100^^^%^^^"'); +}); diff --git a/apps/p2/test/p2/mcp-worker.test.ts b/apps/p2/test/p2/mcp-worker.test.ts new file mode 100644 index 000000000..7a4d33f91 --- /dev/null +++ b/apps/p2/test/p2/mcp-worker.test.ts @@ -0,0 +1,108 @@ +// MCP tools in a session worker, end to end with the fake provider: the worker leases the owner's servers at the +// session's location with the project `.pi/mcp.json` it read there, `tool_search` loads the deferred MCP tool, the +// agent's call returns the server's answer, a relocation moves the lease, and a subagent's own location is leased +// while it is not archived. +import assert from "node:assert/strict"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import type { ToolResultMessage } from "@earendil-works/pi-ai"; +import type { ConversationView } from "@earendil-works/pi-durable"; +import { BACKGROUND_CONTEXT } from "@p2/app/v1"; +import { Chat } from "../../plugins/chat/contract.ts"; +import { Mcp } from "../../plugins/mcp/contract.ts"; +import { Sessions } from "../../plugins/sessions/contract.ts"; +import { holdIn, startFixture, until } from "./helpers.ts"; + +const FIXTURE = fileURLToPath(new URL("../fixtures/mcp-server.mjs", import.meta.url)); + +const results = (view: ConversationView | null | undefined): ToolResultMessage[] => + (view?.entries ?? []).flatMap((entry) => + (entry.model ?? []).filter((message): message is ToolResultMessage => message.role === "toolResult"), + ); +const text = (result: ToolResultMessage | undefined) => + (result?.content ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join(""); + +test("a session's agent finds an MCP tool with tool_search and calls it", { timeout: 90_000 }, async (t) => { + const fixture = await startFixture(); + t.after(() => fixture.close()); + writeFileSync( + join(fixture.dataDir, "mcp.json"), + JSON.stringify({ mcpServers: { files: { command: process.execPath, args: [FIXTURE] } } }), + ); + // The project at the session's cwd adds a server there and refuses an HTTP one. + mkdirSync(join(fixture.workspace, ".pi")); + writeFileSync( + join(fixture.workspace, ".pi", "mcp.json"), + JSON.stringify({ + mcpServers: { + lint: { command: process.execPath, args: [FIXTURE] }, + remote: { url: "https://example.com/mcp" }, + }, + }), + ); + const owner = fixture.browser(); + const mcp = await holdIn(owner, Mcp, {}, "owner"); + const sessions = await holdIn(owner, Sessions, {}, "owner"); + const session = await sessions.create({ cwd: fixture.workspace, title: "MCP" }, BACKGROUND_CONTEXT); + const chat = await holdIn(owner, Chat, { session: session.id, conversation: 1 }, "participant"); + // The worker leases the session's location; the owner's and the project's servers connect there. + const started = await until( + mcp.instances, + (instances) => + instances.length === 2 && + instances.every((item) => item.status === "connected" && item.location?.cwd === fixture.workspace), + ); + assert.deepEqual( + started.map((item) => [item.server, item.source]), + [ + ["files", "owner"], + ["lint", "project"], + ], + ); + const instance = started[0]; + const [refused] = await until(mcp.errors, (errors) => errors.length > 0); + assert.match(refused!, /\.pi\/mcp\.json: server "remote": HTTP servers are for the owner's mcp\.json/); + let serial = 0; + const call = async (prompt: string) => { + const count = results(await until(chat.view, Boolean)).length; + await chat.send({ text: prompt, requestId: `mcp-${++serial}` }, BACKGROUND_CONTEXT); + return results( + await until( + chat.view, + (view) => results(view).length > count && !(view?.docs["pi.live"] as { run?: unknown })?.run, + ), + ).at(-1)!; + }; + const found = await call(`tool_search ${JSON.stringify({ query: "echo" })}`); + assert.equal(found.isError, false, text(found)); + assert.match(text(found), /mcp__files__echo/); + const echoed = await call(`mcp__files__echo ${JSON.stringify({ text: "hello" })}`); + assert.equal(echoed.isError, false, text(echoed)); + assert.equal(text(echoed), "echo: hello"); + assert.deepEqual(echoed.details, { server: "files", tool: "echo" }); + // The session moves in place: the worker leases the new location (no project there) and gives up the old one. + const moved = join(fixture.workspace, "moved"); + mkdirSync(moved); + await sessions.relocate({ id: session.id, cwd: moved, env: null }, BACKGROUND_CONTEXT); + const after = await until( + mcp.instances, + (instances) => + instances.length === 1 && instances[0]!.location?.cwd === moved && instances[0]!.status === "connected", + ); + assert.notEqual(after[0]!.id, instance!.id); + assert.equal(text(await call(`mcp__files__echo ${JSON.stringify({ text: "there" })}`)), "echo: there"); + // A subagent spawned with its own cwd (a commit of the subagent registry): its location is leased too, until it is + // archived. + const own = join(moved, "own"); + mkdirSync(own); + await call( + `subagent ${JSON.stringify({ action: "spawn", name: "own", message: "hi", cwd: "own", tools: ["bash"] })}`, + ); + await until(mcp.instances, (instances) => + instances.some((item) => item.location?.cwd === own && item.status === "connected"), + ); + await call(`subagent ${JSON.stringify({ action: "archive", name: "own" })}`); + await until(mcp.instances, (instances) => instances.length === 1 && instances[0]!.location?.cwd === moved); +}); diff --git a/apps/p2/test/p2/mcp.test.ts b/apps/p2/test/p2/mcp.test.ts new file mode 100644 index 000000000..12083f809 --- /dev/null +++ b/apps/p2/test/p2/mcp.test.ts @@ -0,0 +1,764 @@ +// The MCP manager and its views on a bare server host, with the stdio fixture server (test/fixtures/mcp-server.mjs), a +// small Streamable HTTP server and a stub of the workers' states: instances per session and location or shared, leases +// set per worker generation and released by the view's end, processes ending with their instances, edits of mcp.json, +// the generation and incarnation fences, placement refusals, projects' mcp.json, where sessions work, environments +// through a local pi-env daemon, a reload, guests. Reconnects: mcp-instances.test.ts. +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { homedir, tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Connection } from "@earendil-works/pi-env"; +import { constant, follow, type Location, type P2Principal } from "@p2/app/v1"; +import { Hosts, Server, type WorkerState } from "@p2/app/v1/server"; +import type { EnvConfig } from "../../plugins/envs/config.ts"; +import { EnvService } from "../../plugins/envs/contract.ts"; +import { expandHostVariables, parseOwnerConfig } from "../../plugins/mcp/config.ts"; +import { type Instance, Mcp, McpManager } from "../../plugins/mcp/contract.ts"; +import type { ConnectEnv } from "../../plugins/mcp/environments.ts"; +import { mcpServerFacet } from "../../plugins/mcp/server.ts"; +import { Catalog, type Session } from "../../plugins/sessions/contract.ts"; +import { type Child, ChildStore } from "../../plugins/subagents/contract.ts"; +import { APP, type Cell, expectChord, failureOf, holdIn, poll, settled, TEST, testHost, until } from "./helpers.ts"; + +const FIXTURE = fileURLToPath(new URL("../fixtures/mcp-server.mjs", import.meta.url)); +/** The pi-env daemon environments start servers through: `PI_ENV_DAEMON`, else the debug build of packages/env. */ +const DAEMON = + process.env.PI_ENV_DAEMON ?? + fileURLToPath( + new URL( + `../../../../packages/env/daemon/target/debug/pi-env${process.platform === "win32" ? ".exe" : ""}`, + import.meta.url, + ), + ); +const OWNER: P2Principal = { kind: "owner", access: "local" }; +const GUEST: P2Principal = { + kind: "guest", + id: "g1", + name: "Guest", + home: "s1", + sessions: { s1: "full" }, + revision: 1, +}; + +type Paths = { readonly root: string; readonly log: string }; + +/** Views linger this long after their last reference here (30 s in p2). */ +const LINGER_MS = 150; +/** Environments of envs.json: none unless a test configures them. */ +const NO_ENVIRONMENTS: ConnectEnv = () => { + throw new Error("no environments in this test"); +}; +const mcpServer = mcpServerFacet(LINGER_MS, () => NO_ENVIRONMENTS); + +const UNUSED = (): never => { + throw new Error("unused in these tests"); +}; + +/** A catalog entry of session `id` working on `env` (null: this machine). */ +const session = (id: string, env: string | null): Session => ({ + id, + env, + cwd: "/", + title: id, + createdAt: 0, + updatedAt: 0, + archivedAt: null, + pinned: false, + model: { provider: "faux", modelId: "faux-1" }, + thinkingLevel: "off", +}); + +/** The fixture server as a stdio entry; every start appends its pid to `log`. */ +const stdio = ({ log }: Paths, extra: Record<string, unknown> = {}) => ({ + command: process.execPath, + args: [FIXTURE], + env: { MCP_FIXTURE_LOG: log }, + ...extra, +}); + +/** + * A server host with the MCP plugin and `mcp.json` holding `servers(paths)`, in a fresh data directory. Workers taken + * with `worker` are published running in the stub of the workers' states first, their sessions in the stub of the + * catalog. `options.envs` are envs.json's environments, reached through `options.facet`'s connector. + */ +async function mcpHost( + t: { after(fn: () => unknown): void }, + servers: (paths: Paths) => Record<string, unknown>, + options: { envs?: Record<string, EnvConfig>; facet?: typeof mcpServer } = {}, +) { + const root = realpathSync(mkdtempSync(join(tmpdir(), "p2-mcp-"))); + const dataDir = join(root, "data"); + const paths = { root, log: join(root, "starts.log") }; + mkdirSync(dataDir); + const configure = (entries: Record<string, unknown>) => + writeFileSync(join(dataDir, "mcp.json"), JSON.stringify({ mcpServers: entries })); + configure(servers(paths)); + let workerStates: Cell<Record<string, WorkerState>> | undefined; + let sessions: Cell<Session[]> | undefined; + let children: Cell<Record<string, Child[]>> | undefined; + const fixture = await testHost({ + facets: [options.facet ?? mcpServer], + stubs: [ + // What the manager reads of these: envs.json, sessions' and subagents' locations; their methods are unused. + [ + EnvService, + { + configs: constant(options.envs ?? {}), + tests: constant({}), + failure: constant(null), + test: UNUSED, + reload: UNUSED, + }, + ], + [ + Catalog, + (_params, _principal, self) => { + sessions = self.state<Session[]>([]); + return { sessions, get: UNUSED, create: UNUSED, activity: UNUSED, update: UNUSED }; + }, + ], + [ + ChildStore, + (_params, _principal, self) => { + children = self.state<Record<string, Child[]>>({}); + return { children, report: UNUSED }; + }, + ], + [ + Server, + { + info: constant({ app: APP, dataDir, fake: true, allowedHosts: [], port: 0 }), + revokeOwners: () => {}, + revokeGuests: () => {}, + }, + ], + [ + Hosts, + (_params, _principal, self) => { + workerStates = self.state<Record<string, WorkerState>>({}); + return { workers: workerStates, stop: () => {} }; + }, + ], + ], + }); + const manager = await fixture.host.get(McpManager, {}, TEST); + const workers: { release(): void }[] = []; + let closed = false; + const shutdown = async () => { + closed = true; + await fixture.close(); + }; + t.after(async () => { + if (!closed) { + // Workers go first, as when their processes exit: their views release the leases while the manager lives. + for (const ref of workers) ref.release(); + await until(manager.leases, (leases) => leases.length === 0); + await shutdown(); + } + rmSync(root, { recursive: true, force: true }); + }); + const running = (session: string, generation: number) => + workerStates!.change(TEST, (draft) => { + draft[session] = { running: true, generation }; + }); + /** Records `id` in the catalog as working on `env`, unless it is there already. */ + const listed = (id: string, env: string | null) => + sessions!.value.some((entry) => entry.id === id) + ? Promise.resolve() + : sessions!.change(TEST, (draft) => { + draft.push(session(id, env)); + }); + return { + fixture, + manager, + shutdown, + /** Publishes `session`'s worker `generation` as running, the session in the catalog (on this machine) if new. */ + running: async (session: string, generation: number) => { + await listed(session, null); + await running(session, generation); + }, + /** Moves session `id` to `env` in the catalog. */ + move: (id: string, env: string | null) => + sessions!.change(TEST, (draft) => { + for (const entry of draft) if (entry.id === id) entry.env = env; + }), + /** Records `id`'s subagents (their own locations). */ + subagents: (id: string, list: Child[]) => + children!.change(TEST, (draft) => { + draft[id] = list; + }), + configure: (next: (paths: Paths) => Record<string, unknown>) => configure(next(paths)), + /** Writes `text` as mcp.json as it is (a save caught half-way). */ + write: (text: string) => writeFileSync(join(dataDir, "mcp.json"), text), + /** A directory on this machine as a location. */ + at: (name: string): Location => { + const cwd = join(root, name); + mkdirSync(cwd, { recursive: true }); + return { env: null, cwd }; + }, + /** Fixture process starts so far. */ + starts: () => (existsSync(paths.log) ? readFileSync(paths.log, "utf8").trim().split("\n").filter(Boolean) : []), + owner: () => holdIn(fixture.client(OWNER), Mcp, {}, "owner"), + /** + * The view of `session`'s worker `generation` (published as the session's running worker unless `stale`), the + * session working on `env` (default this machine) unless the catalog lists it already. + */ + worker: async (session: string, generation = 1, options: { stale?: boolean; env?: string | null } = {}) => { + await listed(session, options.env ?? null); + if (!options.stale) await running(session, generation); + const ref = await holdIn(fixture.client({ kind: "worker", session, generation }), Mcp, {}, "worker"); + workers.push(ref); + return ref; + }, + }; +} + +type WorkerView = Awaited<ReturnType<Awaited<ReturnType<typeof mcpHost>>["worker"]>>; + +const textOf = (result: { content: readonly { type: string; text?: string }[] }) => result.content[0]?.text ?? ""; +const status = (instances: readonly Instance[], server: string) => + instances.filter((instance) => instance.server === server).map((instance) => instance.status); +const isUp = + (server: string, count = 1) => + (instances: readonly Instance[]) => + status(instances, server).filter((value) => value === "connected").length === count; +/** The fixture's own report: its pid, its parent's, cwd, the client's roots and its `MCP_FIXTURE_*` variables. */ +const whoami = async (view: WorkerView, location: Location, server: string) => + JSON.parse(textOf(await view.call({ location, server, tool: "whoami", args: {} }, TEST))) as { + pid: number; + ppid: number; + cwd: string; + roots: { uri: string }[] | null; + vars: Record<string, string>; + }; +const alive = (pid: number) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; +const exited = (pid: number) => poll(() => !alive(pid), 10_000, `process ${pid} to exit`); + +test("owner config defaults: stdio per session at the location, shared on the host, HTTP shared", () => { + const { servers, errors } = parseOwnerConfig( + JSON.stringify({ + mcpServers: { + files: { command: "npx", args: ["server"] }, + browser: { command: "npx", shared: true }, + gpu: { command: "/opt/tool", runOn: { env: "gpu" } }, + docs: { url: "https://example.com/mcp" }, + mine: { url: "https://example.com/mcp", shared: false }, + "a-b": { command: "x" }, + a_b: { command: "y" }, + }, + }), + ); + assert.deepEqual( + servers.map((server) => [server.name, server.shared, server.kind === "stdio" ? server.runOn : "http"]), + [ + ["files", false, "location"], + ["browser", true, "host"], + ["gpu", false, { env: "gpu" }], + ["docs", true, "http"], + ["mine", false, "http"], + ["a-b", false, "location"], + ], + ); + assert.deepEqual(errors, ['server "a_b": its tools would be named like those of "a-b"']); + assert.throws(() => parseOwnerConfig("{"), /Invalid mcp.json syntax/); + process.env.P2_MCP_TEST_VALUE = "v"; + assert.equal(expandHostVariables(`a \${P2_MCP_TEST_VALUE} $HOME`), "a v $HOME"); + delete process.env.P2_MCP_TEST_VALUE; + assert.throws(() => expandHostVariables(`\${P2_MCP_TEST_VALUE}`), /P2_MCP_TEST_VALUE is not set/); +}); + +test("owner config: valid servers load, invalid entries are reported and skipped", async (t) => { + const host = await mcpHost(t, (paths) => ({ + good: stdio(paths), + "bad name!": { command: "x" }, + nothing: {}, + cimd: { url: "https://example.com/mcp", oauth: { clientRegistration: "cimd" } }, + runOnHttp: { url: "https://example.com/mcp", runOn: "host" }, + sharedHere: { command: "x", shared: true, runOn: "location" }, + off: { command: "x", enabled: false }, + })); + const owner = await host.owner(); + const errors = (await until(owner.errors, (value) => value.length > 0)).join("\n"); + assert.equal(owner.errors.value.length, 5, errors); + assert.match(errors, /invalid server name/); + assert.match(errors, /needs either "command"/); + assert.match(errors, /cimd/); + assert.match(errors, /runOn is for stdio servers/); + assert.match(errors, /shared server cannot run where each conversation works/); + // Nothing runs before a session leases it; then only the valid, enabled server. + assert.deepEqual(owner.instances.value, []); + const s1 = await host.worker("s1"); + await s1.setLocations({ locations: [host.at("one")] }, TEST); + const servers = await until(s1.servers, isUp("good")); + assert.deepEqual( + servers.map((instance) => instance.server), + ["good"], + ); +}); + +test("unshared servers run per session and location, shared ones once; roots and calls", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths), browser: stdio(paths, { shared: true }) })); + const one = host.at("one"); + const two = host.at("two"); + const s1 = await host.worker("s1"); + const s2 = await host.worker("s2"); + await s1.setLocations({ locations: [one] }, TEST); + await s2.setLocations({ locations: [two] }, TEST); + await until(s1.servers, (value) => isUp("files")(value) && isUp("browser")(value)); + const owner = await host.owner(); + const all = await until(owner.instances, (value) => isUp("files", 2)(value) && isUp("browser")(value)); + assert.equal(all.length, 3); + assert.equal(host.starts().length, 3); + const files = all.find((instance) => instance.server === "files" && instance.location?.cwd === one.cwd)!; + assert.deepEqual(files.runsOn, "host"); + assert.deepEqual( + files.tools.map((tool) => tool.name), + ["echo", "whoami", "crash", "grow"], + ); + assert.equal(all.find((instance) => instance.server === "browser")!.location, null); + // Each session's own process, at its location with its cwd as the root; the shared one has no roots. + const first = await whoami(s1, one, "files"); + assert.notEqual(first.pid, (await whoami(s2, two, "files")).pid); + assert.equal(first.cwd, one.cwd); + assert.deepEqual(first.roots, [{ uri: pathToFileURL(one.cwd).href }]); + assert.equal((await whoami(s1, one, "browser")).pid, (await whoami(s2, two, "browser")).pid); + assert.equal((await whoami(s1, one, "browser")).roots, null); + assert.equal( + textOf(await s1.call({ location: one, server: "files", tool: "echo", args: { text: "hi" } }, TEST)), + "echo: hi", + ); + // Another session's location is not leased by this worker. + await assert.rejects(s1.call({ location: two, server: "files", tool: "echo", args: {} }, TEST), /No MCP servers/); +}); + +test("processes end with their instances: a location left out, the view's end, the host's shutdown", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths), browser: stdio(paths, { shared: true }) })); + const one = host.at("one"); + const two = host.at("two"); + const owner = await host.owner(); + const s1 = await host.worker("s1"); + await s1.setLocations({ locations: [one, two] }, TEST); + await until(owner.instances, (value) => isUp("files", 2)(value) && isUp("browser")(value)); + const atOne = (await whoami(s1, one, "files")).pid; + const atTwo = (await whoami(s1, two, "files")).pid; + const shared = (await whoami(s1, one, "browser")).pid; + await s1.setLocations({ locations: [one] }, TEST); + await until(owner.instances, (value) => status(value, "files").length === 1); + await exited(atTwo); + // The worker's reference goes (its link closed): the view ends and releases its leases. + s1.release(); + const left = await until(owner.instances, (value) => status(value, "files").length === 0); + assert.deepEqual(status(left, "browser"), ["connected"]); + await exited(atOne); + assert.equal(alive(shared), true); + await host.shutdown(); + await exited(shared); +}); + +test("editing mcp.json restarts a changed server and closes a removed one", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths), docs: stdio(paths) })); + const one = host.at("one"); + const s1 = await host.worker("s1"); + await s1.setLocations({ locations: [one] }, TEST); + const before = await until(s1.servers, (value) => isUp("files")(value) && isUp("docs")(value)); + const files = (await whoami(s1, one, "files")).pid; + const docs = (await whoami(s1, one, "docs")).pid; + host.configure((paths) => ({ files: stdio(paths, { timeout: 30 }) })); + const after = await until( + s1.servers, + (value) => + value.length === 1 && isUp("files")(value) && value[0]!.id !== before.find((i) => i.server === "files")!.id, + ); + assert.equal(after[0]!.server, "files"); + await exited(files); + await exited(docs); + assert.notEqual((await whoami(s1, one, "files")).pid, files); +}); + +test("generations: only the running current one sets and calls; an older view's end leaves the newer lease", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + const older = await host.worker("s1", 1); + await older.setLocations({ locations: [one] }, TEST); + const [instance] = await until(older.servers, isUp("files")); + const pid = (await whoami(older, one, "files")).pid; + // The session's worker restarts: generation 2 runs now. + const newer = await host.worker("s1", 2); + await expectChord(older.setLocations({ locations: [one] }, TEST), "forbidden"); + await expectChord(older.call({ location: one, server: "files", tool: "echo", args: {} }, TEST), "forbidden"); + await newer.setLocations({ locations: [one] }, TEST); + assert.equal((await until(newer.servers, isUp("files")))[0]!.id, instance!.id); + assert.equal((await whoami(newer, one, "files")).pid, pid); + // The older view really ends (another principal: its own object), and releases only its own generation's leases. + older.release(); + await until( + host.fixture.host.references(Mcp), + (references) => !references.some((ref) => ref.principal?.kind === "worker" && ref.principal.generation === 1), + ); + const leases = await settled(host.manager.leases); + assert.deepEqual( + leases.map((lease) => [lease.session, lease.location.cwd]), + [["s1", one.cwd]], + ); + assert.equal(alive(pid), true); + assert.equal(host.starts().length, 1); + // The manager's own rule, decided on its line: of two generations' concurrent calls, the newer one's leases stay. + const caller = (generation: number) => ({ session: "s9", generation, incarnation: `i${generation}` }); + await Promise.allSettled([ + host.manager.setLocations({ ...caller(4), locations: [one] }, TEST), + host.manager.setLocations({ ...caller(3), locations: [host.at("three")] }, TEST), + ]); + assert.deepEqual( + host.manager.leases.value.filter((lease) => lease.session === "s9").map((lease) => lease.location.cwd), + [one.cwd], + ); + await expectChord(host.manager.setLocations({ ...caller(3), locations: [one] }, TEST), "forbidden"); + await host.manager.setLocations({ ...caller(4), locations: [] }, TEST); +}); + +test("a worker asking before its start is published waits: then it sets, or a newer generation refuses it", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + const early = await host.worker("s1", 1, { stale: true }); + let settledFirst = false; + const first = early.setLocations({ locations: [one] }, TEST).finally(() => { + settledFirst = true; + }); + await settled(host.manager.leases); + assert.equal(settledFirst, false); + await host.running("s1", 1); + await first; + await until(early.servers, isUp("files")); + // Another session's worker waits, and the generation published instead is newer. + const late = await host.worker("s2", 1, { stale: true }); + const refused = late.setLocations({ locations: [one] }, TEST); + await host.running("s2", 2); + await expectChord(refused, "forbidden"); + // Releasing the reference ends a waiting call. + const gone = await host.worker("s3", 1, { stale: true }); + const waiting = gone.setLocations({ locations: [one] }, TEST); + gone.release(); + await assert.rejects(waiting); + assert.deepEqual( + (await settled(host.manager.leases)).map((lease) => lease.session), + ["s1"], + ); +}); + +test("an invalid mcp.json keeps the last valid servers, also across a reload", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + const s1 = await host.worker("s1"); + await s1.setLocations({ locations: [one] }, TEST); + const [instance] = await until(s1.servers, isUp("files")); + const pid = (await whoami(s1, one, "files")).pid; + host.write("{ half saved"); + const owner = await host.owner(); + await until(owner.errors, (errors) => errors.some((error) => /keeping the last valid servers/.test(error))); + await host.fixture.reload([{ ...mcpServer }]); + const [after] = await until(s1.servers, (value) => isUp("files")(value) && host.starts().length === 2); + assert.equal(after!.id, instance!.id); + assert.equal(alive(pid), false); + assert.match(owner.errors.value.join("\n"), /Invalid mcp.json syntax; keeping the last valid servers/); +}); + +test("a view outlives its last reference by the linger: a worker re-taking it keeps its processes", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + await host.running("s1", 1); + // One worker host (one incarnation) whose object lets go of the view and takes it again. + const worker = host.fixture.client({ kind: "worker", session: "s1", generation: 1 }); + const first = await holdIn(worker, Mcp, {}, "worker"); + await first.setLocations({ locations: [one] }, TEST); + await until(first.servers, isUp("files")); + const pid = (await whoami(first, one, "files")).pid; + first.release(); + const again = await holdIn(worker, Mcp, {}, "worker"); + assert.equal((await whoami(again, one, "files")).pid, pid); + assert.equal(host.starts().length, 1); + again.release(); +}); + +test("another incarnation of the same generation takes the lease; the first one's calls are refused", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + // Two hosts with one principal: two incarnations of one worker generation (one view object). + const first = await host.worker("s1", 2); + await first.setLocations({ locations: [one] }, TEST); + await until(first.servers, isUp("files")); + const second = await host.worker("s1", 2); + await second.setLocations({ locations: [one] }, TEST); + await expectChord( + first.call({ location: one, server: "files", tool: "echo", args: { text: "x" } }, TEST), + "forbidden", + ); + assert.equal( + textOf(await second.call({ location: one, server: "files", tool: "echo", args: { text: "x" } }, TEST)), + "echo: x", + ); +}); + +test("placement: environments not in envs.json fail; HTTP servers connect with host variables, per session when unshared, or ask to sign in", async (t) => { + const authorizations: (string | undefined)[] = []; + const sessions = new Set<string>(); + const http = createServer((request, response) => { + if (request.url === "/locked") return void response.writeHead(401).end(); + if (request.method !== "POST") return void response.writeHead(405).end(); + authorizations.push(request.headers.authorization); + let body = ""; + request.on("data", (chunk: Buffer) => { + body += chunk.toString(); + }); + request.on("end", () => { + const message = JSON.parse(body) as { id?: number; method: string }; + if (message.id === undefined) return void response.writeHead(202).end(); + const result = + message.method === "initialize" + ? { + protocolVersion: "2025-06-18", + capabilities: { tools: {} }, + serverInfo: { name: "http-fixture", version: "1.0.0" }, + } + : message.method === "tools/list" + ? { tools: [{ name: "lookup", inputSchema: { type: "object" } }] } + : { content: [{ type: "text", text: "found" }] }; + // Each `initialize` opens one MCP session: one per connection of the client. + const session = message.method === "initialize" ? `m${sessions.size}` : undefined; + if (session !== undefined) sessions.add(`${request.url}:${session}`); + response.writeHead(200, { + "content-type": "application/json", + ...(session === undefined ? {} : { "mcp-session-id": session }), + }); + response.end(JSON.stringify({ jsonrpc: "2.0", id: message.id, result })); + }); + }); + await new Promise<void>((resolve) => http.listen(0, "127.0.0.1", resolve)); + t.after(() => new Promise((resolve) => http.close(resolve))); + const base = `http://127.0.0.1:${(http.address() as AddressInfo).port}`; + process.env.P2_MCP_TEST_TOKEN = "secret-token"; + t.after(() => delete process.env.P2_MCP_TEST_TOKEN); + const host = await mcpHost(t, (paths) => ({ + here: stdio(paths), + gpu: stdio(paths, { runOn: { env: "gpu" } }), + docs: { url: `${base}/mcp`, headers: { Authorization: `Bearer \${P2_MCP_TEST_TOKEN}` } }, + mine: { url: `${base}/mine`, shared: false }, + locked: { url: `${base}/locked` }, + })); + const remote = { env: "remote", cwd: "/work" }; + const s1 = await host.worker("s1", 1, { env: "remote" }); + const s2 = await host.worker("s2"); + await s1.setLocations({ locations: [remote] }, TEST); + await s2.setLocations({ locations: [host.at("two")] }, TEST); + const servers = await until( + s1.servers, + (value) => + value.length === 5 && isUp("docs")(value) && isUp("mine")(value) && status(value, "locked")[0] === "sign-in", + ); + for (const server of ["here", "gpu"]) { + const instance = servers.find((candidate) => candidate.server === server)!; + assert.equal(instance.status, "failed"); + // The worker's view leaves the owner's error out. + assert.equal(instance.error, undefined); + } + assert.deepEqual(servers.find((instance) => instance.server === "here")!.runsOn, { env: "remote" }); + assert.deepEqual(servers.find((instance) => instance.server === "gpu")!.runsOn, { env: "gpu" }); + const owner = await host.owner(); + const all = await until(owner.instances, (value) => isUp("mine", 2)(value) && isUp("docs")(value)); + assert.deepEqual( + all.filter((instance) => instance.server === "here" && instance.location?.env === "remote").map((i) => i.error), + ['environment "remote" is not in envs.json'], + ); + // One shared docs connection for both sessions, one unshared one per session. + assert.equal(status(all, "docs").length, 1); + assert.equal([...sessions].filter((session) => session.startsWith("/mine:")).length, 2); + assert.equal(host.starts().length, 0); + assert.ok(authorizations.includes("Bearer secret-token")); + assert.equal(textOf(await s1.call({ location: remote, server: "docs", tool: "lookup", args: {} }, TEST)), "found"); +}); + +test("a project's mcp.json: its servers run at its location for its session, it turns the owner's on and off there, refusals are reported", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths), off: stdio(paths, { enabled: false }) })); + const one = host.at("one"); + const two = host.at("two"); + const project = { + location: one, + text: JSON.stringify({ + mcpServers: { + lint: { command: process.execPath, args: [FIXTURE], env: { MCP_FIXTURE_RAW: `\${HOME}` } }, + files: { enabled: false }, + off: { enabled: true }, + remote: { url: "https://example.com/mcp" }, + }, + }), + }; + const s1 = await host.worker("s1"); + const s2 = await host.worker("s2"); + await s1.setLocations( + { locations: [one, two], projects: [project, { location: two, error: "permission denied" }] }, + TEST, + ); + await s2.setLocations({ locations: [one] }, TEST); + const servers = await until( + s1.servers, + (value) => value.length === 3 && value.every((i) => i.status === "connected"), + ); + assert.deepEqual( + servers.map((instance) => [instance.server, instance.source, instance.location?.cwd, instance.runsOn]), + [ + ["files", "owner", two.cwd, "host"], + ["lint", "project", one.cwd, "host"], + ["off", "owner", one.cwd, "host"], + ], + ); + // The project's server runs at its location, its variables as written. + const lint = await whoami(s1, one, "lint"); + assert.equal(lint.cwd, one.cwd); + assert.equal(lint.vars.MCP_FIXTURE_RAW, `\${HOME}`); + await assert.rejects( + s1.call({ location: one, server: "files", tool: "echo", args: {} }, TEST), + /MCP server "files" is not configured at/, + ); + const errors = s1.errors.value.join("\n"); + assert.equal(s1.errors.value.length, 2, errors); + assert.match(errors, /: \.pi\/mcp\.json: server "remote": HTTP servers are for the owner's mcp\.json/); + assert.match(errors, /: \.pi\/mcp\.json: permission denied/); + // Another session at the same location without a project: the owner's servers only; the owner sees every error. + assert.deepEqual( + (await until(s2.servers, isUp("files"))).map((instance) => instance.server), + ["files"], + ); + const owner = await host.owner(); + assert.deepEqual( + (await until(owner.errors, (value) => value.length === 2)).map((error) => error.startsWith("session s1: ")), + [true, true], + ); + // The project file gone: its server ends, the owner's servers are as configured again. + await s1.setLocations({ locations: [one] }, TEST); + await until(s1.servers, (value) => value.length === 1 && isUp("files")(value)); + await exited(lint.pid); + assert.deepEqual(await until(s1.errors, (value) => value.length === 0), []); +}); + +test("a lease gets servers only on environments its session works on: its own, its subagents' own locations'", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + const gpu = { env: "gpu", cwd: "/work" }; + const s1 = await host.worker("s1"); + await s1.setLocations({ locations: [one, gpu] }, TEST); + const [here] = await until(s1.servers, isUp("files")); + assert.equal(here!.location?.cwd, one.cwd); + assert.deepEqual(s1.errors.value, ["gpu:/work: not where this session works"]); + // A subagent working on gpu in its own location: the lease there gets its servers (gpu is not in envs.json). + const child = { + name: "own", + conversationId: 2, + model: "faux", + thinkingLevel: "off", + tools: [], + location: { env: "gpu", cwd: "/elsewhere" }, + lastEntryId: 0, + }; + await host.subagents("s1", [child]); + const both = await until(s1.servers, (value) => value.length === 2); + assert.deepEqual(both.map((instance) => [instance.location?.env ?? "this machine", instance.status]).sort(), [ + ["gpu", "failed"], + ["this machine", "connected"], + ]); + assert.deepEqual(await until(s1.errors, (value) => value.length === 0), []); + // The subagent gone and the session moved to gpu: this machine is no longer where it works. + await host.subagents("s1", []); + await host.move("s1", "gpu"); + const moved = await until(s1.servers, (value) => value.length === 1 && value[0]!.location?.env === "gpu"); + assert.equal(moved[0]!.status, "failed"); + assert.deepEqual(s1.errors.value, [`${one.cwd}: not where this session works`]); +}); + +test( + "environments: servers start through the environment's pi-env daemon, on one connection that closes with the last instance", + { skip: existsSync(DAEMON) ? false : `no pi-env daemon at ${DAEMON} (npm run build:daemon in packages/env)` }, + async (t) => { + // The "environment" is this machine through a local daemon: no SSH, the same operations. + const connections: Connection[] = []; + const facet = mcpServerFacet(LINGER_MS, () => () => { + const connection = new Connection({ command: [DAEMON] }); + connections.push(connection); + return connection; + }); + const host = await mcpHost( + t, + (paths) => ({ files: stdio(paths), tools: stdio(paths, { runOn: { env: "box" }, cwd: "~" }) }), + { envs: { box: { type: "ssh", host: "box.invalid" } }, facet }, + ); + const there = { env: "box", cwd: host.at("work").cwd }; + const s1 = await host.worker("s1", 1, { env: "box" }); + await s1.setLocations({ locations: [there] }, TEST); + const servers = await until(s1.servers, (value) => isUp("files")(value) && isUp("tools")(value)); + assert.deepEqual( + servers.map((instance) => [instance.server, instance.runsOn]), + [ + ["files", { env: "box" }], + ["tools", { env: "box" }], + ], + ); + assert.equal(connections.length, 1); + const daemon = (await connections[0]!.info()).pid; + // Both run under the one daemon with the location as their root: `files` at the location, `tools` at its cwd `~`, + // the environment's home. + const files = await whoami(s1, there, "files"); + const tools = await whoami(s1, there, "tools"); + assert.deepEqual([files.ppid, tools.ppid], [daemon, daemon]); + assert.equal(files.cwd, there.cwd); + assert.deepEqual(files.roots, [{ uri: pathToFileURL(there.cwd).href }]); + assert.equal(tools.cwd, homedir()); + assert.deepEqual(tools.roots, files.roots); + assert.equal( + textOf(await s1.call({ location: there, server: "files", tool: "echo", args: { text: "far" } }, TEST)), + "echo: far", + ); + // The last instance there closes the connection: the daemon ends, and the servers with it. + await s1.setLocations({ locations: [] }, TEST); + await exited(files.pid); + await exited(tools.pid); + await exited(daemon); + assert.equal(connections.length, 1); + }, +); + +test("a reload of the plugin keeps the leases and the tools: the new supervisor starts what they use", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const one = host.at("one"); + const s1 = await host.worker("s1"); + await s1.setLocations({ locations: [one] }, TEST); + const [before] = await until(s1.servers, isUp("files")); + let fewest = before!.tools.length; + const stop = follow(s1.servers, (servers) => { + fewest = Math.min(fewest, servers[0]?.tools.length ?? 0); + }); + // A new facet value: a reload, not the same code again. + await host.fixture.reload([{ ...mcpServer }]); + const [after] = await until(s1.servers, (value) => isUp("files")(value) && host.starts().length === 2); + stop(); + assert.equal(after!.id, before!.id); + assert.equal(fewest, 4); + assert.equal( + textOf(await s1.call({ location: one, server: "files", tool: "echo", args: { text: "again" } }, TEST)), + "echo: again", + ); +}); + +test("guests get no MCP servers", async (t) => { + const host = await mcpHost(t, (paths) => ({ files: stdio(paths) })); + const error = await failureOf(host.fixture.client(GUEST).hold(Mcp, {}, TEST)); + assert.equal(error.code, "forbidden"); +}); diff --git a/apps/p2/test/p2/plugin-layout.test.ts b/apps/p2/test/p2/plugin-layout.test.ts index 8eda6b1a6..d27422307 100644 --- a/apps/p2/test/p2/plugin-layout.test.ts +++ b/apps/p2/test/p2/plugin-layout.test.ts @@ -29,8 +29,11 @@ const SHARED = [ "@earendil-works/pi-durable/codemode", "@earendil-works/pi-durable/env", "@earendil-works/pi-durable/env/node", + "@earendil-works/pi-durable/tool-search", "@earendil-works/pi-durable/tools", "@earendil-works/pi-env", + "@earendil-works/pi-mcp", + "@earendil-works/pi-mcp/oauth", "@earendil-works/pi-schema", "@earendil-works/pi-schema/formats", "@earendil-works/pi-tui", diff --git a/package-lock.json b/package-lock.json index a60c9db34..06e7dd2e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,8 @@ "@earendil-works/pi-ai": "1.5.0", "@earendil-works/pi-codemode": "1.5.0", "@earendil-works/pi-durable": "1.5.0", + "@earendil-works/pi-env": "1.5.0", + "@earendil-works/pi-mcp": "1.5.0", "@earendil-works/pi-schema": "1.5.0", "@earendil-works/pi-tui": "1.5.0", "diff": "8.0.4",